Includes new 'cluster init' target; Prints errors to the STDOUT

Signed-off-by: Josh Kneubuhl <jkneubuh@us.ibm.com>

ci/scripts/run-k8s-test-network-basic.sh

@@ -56,6 +56,7 @@ function quitterLaScene() {
 function createCluster() {
   print "Initializing KIND Kubernetes cluster"
   ./network kind
+  ./network cluster init
 }
 
 function destroyCluster() {

test-network-k8s/README.md

@@ -17,30 +17,32 @@ _Fabric, Ahoy!_
 
 ## Prerequisites:
 
-- K8s - either:
-  - [KIND](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) + [Docker](https://www.docker.com) (resources: 8 CPU / 8 GRAM) or
-  - [Rancher Desktop](https://rancherdesktop.io) (resources: 8 CPU / 8GRAM, mobyd, and disable Traefik)
 - [kubectl](https://kubernetes.io/docs/tasks/tools/)
 - [jq](https://stedolan.github.io/jq/)
 - [envsubst](https://www.gnu.org/software/gettext/manual/html_node/envsubst-Invocation.html) (`brew install gettext` on OSX)
 
+- K8s - either:
+  - [KIND](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) + [Docker](https://www.docker.com) (resources: 8 CPU / 8 GRAM) 
+  - [Rancher Desktop](https://rancherdesktop.io) (resources: 8 CPU / 8GRAM, mobyd, and disable Traefik)
 
 ## Quickstart 
 
-Create a KIND Kubernetes:
+Create a KIND cluster:  
 ```shell
 ./network kind
+./network cluster init
 ```
-For environments running [Rancher k3s](docs/KUBERNETES.md#rancher-desktop-and-k3s): 
+or for [Rancher / k3s](docs/KUBERNETES.md#rancher-desktop-and-k3s):
 ```shell
 export TEST_NETWORK_CLUSTER_RUNTIME=k3s
 
-./network cluster-init 
+./network cluster init
 ```
 
 Launch the network, create a channel, and deploy the [basic-asset-transfer](../asset-transfer-basic) smart contract: 
 ```shell
 ./network up
+
 ./network channel create
 
 ./network chaincode deploy asset-transfer-basic basic_1.0 $PWD/../asset-transfer-basic/chaincode-java
@@ -48,7 +50,7 @@ Launch the network, create a channel, and deploy the [basic-asset-transfer](../a
 
 Invoke and query chaincode:
 ```shell
-./network chaincode invoke asset-transfer-basic '{"Args":["CreateAsset","1","blue","35","tom","1000"]}' 
+./network chaincode invoke asset-transfer-basic '{"Args":["InitLedger"]}'
 ./network chaincode query  asset-transfer-basic '{"Args":["ReadAsset","1"]}'
 ```
 
@@ -57,16 +59,21 @@ Access the blockchain with a [REST API](https://github.com/hyperledger/fabric-sa
 ./network rest-easy
 ```
 
-Tear down the test network: 
+Shut down the test network: 
 ```shell
 ./network down 
 ```
 
-Tear down the cluster: 
+Tear down the cluster (KIND): 
 ```shell
 ./network unkind
 ```
 
+For Rancher: Preferences -> Kubernetes Settings -> Reset Kubernetes  OR ...
+```shell
+./network cluster clean
+```
+
 
 ## [Detailed Guides](docs/README.md)
 

test-network-k8s/docs/KUBERNETES.md

@@ -47,6 +47,7 @@ for development, testing, or CI, you can create a new cluster with:
 
 ```shell
 $ ./network kind
+$ ./network cluster init
 ```
 
 By default, `kind` will set the current Kube context to reference the new cluster.  Any
@@ -88,15 +89,18 @@ To run natively on k3s, skip the creation of a KIND cluster and:
 ```shell
 export TEST_NETWORK_CLUSTER_RUNTIME="k3s"
 
-./network cluster-init
+./network cluster init
 ```
 - containerd is also a viable runtime.  When building images for chaincode-as-a-service, the `--namespace k8s.io` 
   argument must be applied to the `nerdctl` CLI.  
 
 - For use with containerd: 
 ```shell
-export CONTAINER_CLI="nerdctl"
+export TEST_NETWORK_CLUSTER_RUNTIME="k3s"
 export TEST_NETWORK_CONTAINER_NAMESPACE="--namespace k8s.io"
+export CONTAINER_CLI="nerdctl"
+
+./network cluster init
 ```
 
 

test-network-k8s/network

@@ -35,6 +35,7 @@ export TEMP_DIR=${PWD}/build
 
 LOG_FILE=${TEST_NETWORK_LOG_FILE:-network.log}
 DEBUG_FILE=${TEST_NETWORK_DEBUG_FILE:-network-debug.log}
+LOG_ERROR_LINES=${TEST_NETWORK_LOG_ERROR_LINES:-1}
 LOCAL_REGISTRY_NAME=${TEST_NETWORK_LOCAL_REGISTRY_NAME:-kind-registry}
 LOCAL_REGISTRY_PORT=${TEST_NETWORK_LOCAL_REGISTRY_PORT:-5000}
 STAGE_DOCKER_IMAGES=${TEST_NETWORK_STAGE_DOCKER_IMAGES:-false}
@@ -79,6 +80,7 @@ function print_help() {
 . scripts/utils.sh
 . scripts/prereqs.sh
 . scripts/kind.sh
+. scripts/cluster.sh
 . scripts/fabric_config.sh
 . scripts/fabric_CAs.sh
 . scripts/test_network.sh
@@ -103,24 +105,17 @@ else
 fi
 
 if [ "${MODE}" == "kind" ]; then
-  log "Initializing KIND cluster \"${CLUSTER_NAME}\":"
+  log "Creating KIND cluster \"${CLUSTER_NAME}\":"
   kind_init
-  log "🏁 - Cluster is ready."
+  log "🏁 - KIND cluster is ready"
-
-elif [ "${MODE}" == "cluster-init" ]; then
-  log "Applying k8s ingress and cert-manager"
-  cluster_init
-  log "🏁 - Cluster is ready"
-
-elif [ "${MODE}" == "load-images" ]; then
-  log "Loading images to KIND:"
-  load_docker_images
-  log "🏁 - Images loaded."
 
 elif [ "${MODE}" == "unkind" ]; then
-  log "Deleting cluster \"${CLUSTER_NAME}\":"
+  log "Deleting KIND cluster \"${CLUSTER_NAME}\":"
   kind_unkind
-  log "🏁 - Cluster is gone."
+  log "🏁 - KIND Cluster is gone."
+
+elif [[ "${MODE}" == "cluster" || "${MODE}" == "k8s" || "${MODE}" == "kube" ]]; then
+  cluster_command_group $@
 
 elif [ "${MODE}" == "up" ]; then
   log "Launching network \"${NETWORK_NAME}\":"
@@ -133,18 +128,7 @@ elif [ "${MODE}" == "down" ]; then
   log "🏁 - Fabric network is down."
 
 elif [ "${MODE}" == "channel" ]; then
-  COMMAND=$1
+  channel_command_group $@
-  shift
-
-  if [ "${COMMAND}" == "create" ]; then
-    log "Creating channel \"${CHANNEL_NAME}\":"
-    channel_up
-    log "🏁 - Channel is ready."
-
-  else
-    print_help
-    exit 1
-  fi
 
 elif [[ "${MODE}" == "chaincode" || "${MODE}" == "cc" ]]; then
   chaincode_command_group $@
@@ -152,11 +136,6 @@ elif [[ "${MODE}" == "chaincode" || "${MODE}" == "cc" ]]; then
 elif [ "${MODE}" == "anchor" ]; then
   update_anchor_peers $@
 
-elif [ "${MODE}" == "load-images-for-rest-easy" ]; then
-  log "Loading images for fabric-rest-sample to KIND:"
-  load_docker_images_for_rest_sample
-  log "🏁 - Images loaded."
-
 elif [ "${MODE}" == "rest-easy" ]; then
   log "Launching fabric-rest-sample application:"
   launch_rest_sample

test-network-k8s/scripts/chaincode.sh

@@ -5,6 +5,67 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+# chaincode "group" commands.  Like "main" for chaincode sub-command group.
+function chaincode_command_group() {
+  #set -x
+
+  COMMAND=$1
+  shift
+
+  if [ "${COMMAND}" == "deploy" ]; then
+    log "Deploying chaincode"
+    deploy_chaincode $@
+    log "🏁 - Chaincode is ready."
+
+  elif [ "${COMMAND}" == "activate" ]; then
+    log "Activating chaincode"
+    activate_chaincode $@
+    log "🏁 - Chaincode is ready."
+
+  elif [ "${COMMAND}" == "package" ]; then
+    log "Packaging chaincode"
+    package_chaincode $@
+    log "🏁 - Chaincode package is ready."
+
+  elif [ "${COMMAND}" == "id" ]; then
+    set_chaincode_id $@
+    log $CHAINCODE_ID
+
+  elif [ "${COMMAND}" == "launch" ]; then
+    log "Launching chaincode services"
+    launch_chaincode $@
+    log "🏁 - Chaincode services are ready"
+
+  elif [ "${COMMAND}" == "install" ]; then
+    log "Installing chaincode for org1"
+    install_chaincode $@
+    log "🏁 - Chaincode is installed"
+
+  elif [ "${COMMAND}" == "approve" ]; then
+    log "Approving chaincode for org1"
+    approve_chaincode $@
+    log "🏁 - Chaincode is approved"
+
+  elif [ "${COMMAND}" == "commit" ]; then
+    log "Committing chaincode for org1"
+    commit_chaincode $@
+    log "🏁 - Chaincode is committed"
+
+  elif [ "${COMMAND}" == "invoke" ]; then
+    invoke_chaincode $@ 2>> ${LOG_FILE}
+
+  elif [ "${COMMAND}" == "query" ]; then
+    query_chaincode $@ >> ${LOG_FILE}
+
+  elif [ "${COMMAND}" == "metadata" ]; then
+    query_chaincode_metadata $@ >> ${LOG_FILE}
+
+  else
+    print_help
+    exit 1
+  fi
+}
+
 # Convenience routine to "do everything" required to bring up a sample CC.
 function deploy_chaincode() {
   local cc_name=$1
@@ -278,63 +339,3 @@ function set_chaincode_id() {
   CHAINCODE_ID=${cc_label}:${cc_sha256}
 }
 
-# chaincode "group" commands.  Like "main" for chaincode sub-command group.
-function chaincode_command_group() {
-  #set -x
-
-  COMMAND=$1
-  shift
-
-  if [ "${COMMAND}" == "deploy" ]; then
-    log "Deploying chaincode"
-    deploy_chaincode $@
-    log "🏁 - Chaincode is ready."
-
-  elif [ "${COMMAND}" == "activate" ]; then
-    log "Activating chaincode"
-    activate_chaincode $@
-    log "🏁 - Chaincode is ready."
-
-  elif [ "${COMMAND}" == "package" ]; then
-    log "Packaging chaincode"
-    package_chaincode $@
-    log "🏁 - Chaincode package is ready."
-
-  elif [ "${COMMAND}" == "id" ]; then
-    set_chaincode_id $@
-    log $CHAINCODE_ID
-
-  elif [ "${COMMAND}" == "launch" ]; then
-    log "Launching chaincode services"
-    launch_chaincode $@
-    log "🏁 - Chaincode services are ready"
-
-  elif [ "${COMMAND}" == "install" ]; then
-    log "Installing chaincode for org1"
-    install_chaincode $@
-    log "🏁 - Chaincode is installed"
-
-  elif [ "${COMMAND}" == "approve" ]; then
-    log "Approving chaincode for org1"
-    approve_chaincode $@
-    log "🏁 - Chaincode is approved"
-
-  elif [ "${COMMAND}" == "commit" ]; then
-    log "Committing chaincode for org1"
-    commit_chaincode $@
-    log "🏁 - Chaincode is committed"
-
-  elif [ "${COMMAND}" == "invoke" ]; then
-    invoke_chaincode $@ 2>> ${LOG_FILE}
-
-  elif [ "${COMMAND}" == "query" ]; then
-    query_chaincode $@ >> ${LOG_FILE}
-
-  elif [ "${COMMAND}" == "metadata" ]; then
-    query_chaincode_metadata $@ >> ${LOG_FILE}
-
-  else
-    print_help
-    exit 1
-  fi
-}

test-network-k8s/scripts/channel.sh

@@ -5,8 +5,24 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+function channel_command_group() {
+  # set -x
+
+  COMMAND=$1
+  shift
+
+  if [ "${COMMAND}" == "create" ]; then
+    log "Creating channel \"${CHANNEL_NAME}\":"
+    channel_up
+    log "🏁 - Channel is ready."
+
+  else
+    print_help
+    exit 1
+  fi
+}
+
 function channel_up() {
-  set -x
 
   register_org_admins
   enroll_org_admins

test-network-k8s/scripts/cluster.sh

@@ -0,0 +1,147 @@
+#!/bin/bash
+#
+# Copyright IBM Corp All Rights Reserved
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# cluster "group" commands.  Like "main" for the fabric-cli "cluster" sub-command
+function cluster_command_group() {
+
+  # Default COMMAND is 'init' if not specified
+  if [ "$#" -eq 0 ]; then
+    COMMAND="init"
+
+  else
+    COMMAND=$1
+    shift
+  fi
+
+  if [ "${COMMAND}" == "init" ]; then
+    log "Initializing K8s cluster"
+    cluster_init
+    log "🏁 - Cluster is ready"
+
+  elif [ "${COMMAND}" == "clean" ]; then
+    log "Cleaning k8s cluster"
+    cluster_clean
+    log "🏁 - Cluster is cleaned"
+
+  elif [ "${COMMAND}" == "load-images" ]; then
+    log "Loading Docker images"
+    load_images
+    log "🏁 - Images are loaded"
+
+  else
+    print_help
+    exit 1
+  fi
+}
+
+function pull_docker_images() {
+  push_fn "Pulling docker images for Fabric ${FABRIC_VERSION}"
+
+  $CONTAINER_CLI pull ${CONTAINER_NAMESPACE} ${FABRIC_CONTAINER_REGISTRY}/fabric-ca:$FABRIC_CA_VERSION
+  $CONTAINER_CLI pull ${CONTAINER_NAMESPACE} ${FABRIC_CONTAINER_REGISTRY}/fabric-orderer:$FABRIC_VERSION
+  $CONTAINER_CLI pull ${CONTAINER_NAMESPACE} ${FABRIC_PEER_IMAGE}
+  $CONTAINER_CLI pull ${CONTAINER_NAMESPACE} couchdb:3.2.1
+
+  $CONTAINER_CLI pull ${CONTAINER_NAMESPACE} ghcr.io/hyperledger/fabric-rest-sample:latest
+  $CONTAINER_CLI pull ${CONTAINER_NAMESPACE} redis:6.2.5
+
+  pop_fn
+}
+
+function cluster_init() {
+
+  apply_nginx_ingress
+  apply_cert_manager
+
+  sleep 2
+
+  wait_for_cert_manager
+  wait_for_nginx_ingress
+  
+  if [ "${STAGE_DOCKER_IMAGES}" == true ]; then
+    pull_docker_images
+    load_docker_images
+    pull_docker_images_for_rest_sample
+    load_docker_images_for_rest_sample
+  fi
+}
+
+function apply_nginx() {
+  apply_nginx_ingress
+  wait_for_nginx_ingress
+}
+
+function apply_nginx_ingress() {
+  push_fn "Launching ${CLUSTER_RUNTIME} ingress controller"
+
+  # 1.1.2 static ingress with modifications to enable ssl-passthrough
+  # k3s : 'cloud'
+  # kind : 'kind'
+  # kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.2/deploy/static/provider/cloud/deploy.yaml
+
+  kubectl apply -f kube/ingress-nginx-${CLUSTER_RUNTIME}.yaml
+
+  pop_fn
+}
+
+function delete_nginx_ingress() {
+  push_fn "Deleting ${CLUSTER_RUNTIME} ingress controller"
+
+  cat kube/ingress-nginx-${CLUSTER_RUNTIME}.yaml | kubectl delete -f -
+
+  pop_fn
+}
+
+function wait_for_nginx_ingress() {
+  push_fn "Waiting for ingress controller"
+
+  kubectl wait --namespace ingress-nginx \
+    --for=condition=ready pod \
+    --selector=app.kubernetes.io/component=controller \
+    --timeout=2m
+
+  pop_fn
+}
+
+function apply_cert_manager() {
+  push_fn "Launching cert-manager"
+
+  # Install cert-manager to manage TLS certificates
+  kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.6.1/cert-manager.yaml
+
+  pop_fn
+}
+
+function delete_cert_manager() {
+  push_fn "Deleting cert-manager"
+
+  # Install cert-manager to manage TLS certificates
+  curl https://github.com/jetstack/cert-manager/releases/download/v1.6.1/cert-manager.yaml | kubectl delete -f -
+
+  pop_fn
+}
+
+function wait_for_cert_manager() {
+  push_fn "Waiting for cert-manager"
+
+  kubectl -n cert-manager rollout status deploy/cert-manager
+  kubectl -n cert-manager rollout status deploy/cert-manager-cainjector
+  kubectl -n cert-manager rollout status deploy/cert-manager-webhook
+
+  pop_fn
+}
+
+function cluster_clean() {
+  delete_nginx_ingress
+  delete_cert_manager
+}
+
+function load_images() {
+  if [ "${CLUSTER_RUNTIME}" == "kind" ]; then
+    kind_load_docker_images
+  fi
+}

test-network-k8s/scripts/fabric_config.sh

@@ -13,6 +13,14 @@ function init_namespace() {
   pop_fn
 }
 
+function delete_namespace() {
+  push_fn "Deleting namespace \"$NS\""
+
+  kubectl delete namespace $NS || true
+
+  pop_fn
+}
+
 function init_storage_volumes() {
   push_fn "Provisioning volume storage"
 

test-network-k8s/scripts/kind.sh

@@ -5,93 +5,6 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-function pull_docker_images() {
-  push_fn "Pulling docker images for Fabric ${FABRIC_VERSION}"
-
-  docker pull ${FABRIC_CONTAINER_REGISTRY}/fabric-ca:$FABRIC_CA_VERSION
-  docker pull ${FABRIC_CONTAINER_REGISTRY}/fabric-orderer:$FABRIC_VERSION
-  docker pull ${FABRIC_PEER_IMAGE}
-  docker pull ${FABRIC_CONTAINER_REGISTRY}/fabric-tools:$FABRIC_VERSION
-  docker pull ghcr.io/hyperledgendary/fabric-ccaas-asset-transfer-basic:latest
-  docker pull couchdb:3.2.1
-
-  pop_fn
-}
-
-function load_docker_images() {
-  push_fn "Loading docker images to KIND control plane"
-
-  kind load docker-image ${FABRIC_CONTAINER_REGISTRY}/fabric-ca:$FABRIC_CA_VERSION
-  kind load docker-image ${FABRIC_CONTAINER_REGISTRY}/fabric-orderer:$FABRIC_VERSION
-  kind load docker-image ${FABRIC_PEER_IMAGE}
-  kind load docker-image ${FABRIC_CONTAINER_REGISTRY}/fabric-tools:$FABRIC_VERSION
-  kind load docker-image ghcr.io/hyperledgendary/fabric-ccaas-asset-transfer-basic:latest
-  kind load docker-image couchdb:3.2.1
-
-  pop_fn
-}
-
-function pull_docker_images_for_rest_sample() {
-  push_fn "Pulling docker images for fabric-rest-sample"
-
-  docker pull ghcr.io/hyperledger/fabric-rest-sample:latest
-  docker pull redis:6.2.5
-
-  pop_fn
-}
-
-function load_docker_images_for_rest_sample() {
-  push_fn "Loading docker images for fabric-rest-sample to KIND control plane"
-
-  kind load docker-image ghcr.io/hyperledgendary/fabric-ccaas-asset-transfer-basic:latest
-  kind load docker-image redis:6.2.5
-
-  pop_fn
-}
-
-function apply_nginx_ingress() {
-  push_fn "Launching ${CLUSTER_RUNTIME} ingress controller"
-
-  # 1.1.2 static ingress with modifications to enable ssl-passthrough
-  # k3s : 'cloud'
-  # kind : 'kind'
-  # kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.2/deploy/static/provider/cloud/deploy.yaml
-
-  kubectl apply -f kube/ingress-nginx-${CLUSTER_RUNTIME}.yaml
-
-  pop_fn
-}
-
-function wait_for_nginx_ingress() {
-  push_fn "Waiting for ingress controller"
-
-  kubectl wait --namespace ingress-nginx \
-    --for=condition=ready pod \
-    --selector=app.kubernetes.io/component=controller \
-    --timeout=90s
-
-  pop_fn
-}
-
-function apply_cert_manager() {
-  push_fn "Launching cert-manager"
-
-  # Install cert-manager to manage TLS certificates
-  kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.6.1/cert-manager.yaml
-
-  pop_fn
-}
-
-function wait_for_cert_manager() {
-  push_fn "Waiting for cert-manager"
-
-  kubectl -n cert-manager rollout status deploy/cert-manager
-  kubectl -n cert-manager rollout status deploy/cert-manager-cainjector
-  kubectl -n cert-manager rollout status deploy/cert-manager-webhook
-
-  pop_fn
-}
-
 function kind_create() {
   push_fn  "Creating cluster \"${CLUSTER_NAME}\""
 
@@ -147,6 +60,20 @@ EOF
   pop_fn
 }
 
+function kind_load_docker_images() {
+  push_fn "Loading docker images to KIND control plane"
+
+  kind load docker-image ${FABRIC_CONTAINER_REGISTRY}/fabric-ca:$FABRIC_CA_VERSION
+  kind load docker-image ${FABRIC_CONTAINER_REGISTRY}/fabric-orderer:$FABRIC_VERSION
+  kind load docker-image ${FABRIC_PEER_IMAGE}
+  kind load docker-image couchdb:3.2.1
+
+  kind load docker-image ghcr.io/hyperledger/fabric-rest-sample:latest
+  kind load docker-image redis:6.2.5
+
+  pop_fn
+}
+
 function launch_docker_registry() {
   push_fn "Launching container registry \"${LOCAL_REGISTRY_NAME}\" at localhost:${LOCAL_REGISTRY_PORT}"
 
@@ -183,48 +110,29 @@ EOF
   pop_fn
 }
 
+function stop_docker_registry() {
+  push_fn "Deleting container registry \"${LOCAL_REGISTRY_NAME}\" at localhost:${LOCAL_REGISTRY_PORT}"
+
+  docker kill kind-registry || true
+  docker rm kind-registry   || true
+
+  pop_fn
+}
+
 function kind_delete() {
   push_fn "Deleting KIND cluster ${CLUSTER_NAME}"
 
   kind delete cluster --name $CLUSTER_NAME
 
-  pop_fn 2
+  pop_fn
 }
 
 function kind_init() {
-  # todo: how to pass this through to push_fn ?
-  set -o errexit
-
   kind_create
-
   launch_docker_registry
-
-  if [ "${STAGE_DOCKER_IMAGES}" == true ]; then
-    pull_docker_images
-    load_docker_images
-    pull_docker_images_for_rest_sample
-    load_docker_images_for_rest_sample
-  fi
-
-  cluster_init
-}
-
-function cluster_init() {
-
-  apply_nginx_ingress
-  apply_cert_manager
-
-  sleep 2
-
-  wait_for_cert_manager
-  wait_for_nginx_ingress
 }
 
 function kind_unkind() {
   kind_delete
-}
+  stop_docker_registry
-
-function apply_nginx() {
-  apply_nginx_ingress
-  wait_for_nginx_ingress
 }

test-network-k8s/scripts/test_network.sh

@@ -224,7 +224,7 @@ function network_down() {
   stop_services
   scrub_org_volumes
 
-  kubectl delete namespace $NS
+  delete_namespace
 
   rm -rf $PWD/build
 }

test-network-k8s/scripts/utils.sh

@@ -26,8 +26,6 @@ function exit_fn() {
   rc=$?
   set +x
 
-  set +x
-
   # Write an error icon to the current logging statement.
   if [ "0" -ne $rc ]; then
     pop_fn $rc
@@ -58,22 +56,26 @@ function pop_fn() {
 
   local res=$1
   if [ $res -eq 0 ]; then
-    echo -ne "\r✅"  >> ${LOG_FILE}
+    echo -ne "\r✅\n"  >> ${LOG_FILE}
 
   elif [ $res -eq 1 ]; then
-    echo -ne "\r⚠️" >> ${LOG_FILE}
+    echo -ne "\r⚠️\n" >> ${LOG_FILE}
 
   elif [ $res -eq 2 ]; then
-    echo -ne "\r☠️" >> ${LOG_FILE}
+    echo -ne "\r☠️\n" >> ${LOG_FILE}
 
   elif [ $res -eq 127 ]; then
-    echo -ne "\r☠️" >> ${LOG_FILE}
+    echo -ne "\r☠️\n" >> ${LOG_FILE}
 
   else
-    echo -ne "\r" >> ${LOG_FILE}
+    echo -ne "\r\n" >> ${LOG_FILE}
   fi
 
-  echo "" >> ${LOG_FILE}
+  if [ $res -ne 0 ]; then
+    tail -${LOG_ERROR_LINES} network-debug.log >> ${LOG_FILE}
+  fi
+
+  #echo "" >> ${LOG_FILE}
 }
 
 # Apply the current environment to a k8s template and apply to the cluster.