Feature/fabric builder k8s (#739)

* Support the fabric-builder-k8s for the chaincode "easy button."

Signed-off-by: Josh Kneubuhl <jkneubuh@us.ibm.com>

* Run the CI/CD test suite using the correct matrix env for k8s builder

Signed-off-by: Josh Kneubuhl <jkneubuh@us.ibm.com>

* Incorporate review feedback : remove zz_unused and pkgk8scc routines

Signed-off-by: Josh Kneubuhl <jkneubuh@us.ibm.com>

ci/azure-pipelines.yml

@@ -121,6 +121,7 @@ jobs:
         Basic-Typescript:
           CHAINCODE_NAME: basic
           CHAINCODE_LANGUAGE: typescript
+
     steps:
       - template: templates/install-deps.yml
       - script: sudo apt-get install softhsm2
@@ -135,12 +136,17 @@ jobs:
       vmImage: ubuntu-20.04
     strategy:
       matrix:
-        Typescript-Java:
+        CCaaS-Java:
           CLIENT_LANGUAGE: typescript
           CHAINCODE_LANGUAGE: java
-        Typescript-Golang:
+        CCaaS-Golang:
           CLIENT_LANGUAGE: typescript
           CHAINCODE_LANGUAGE: external
+        K8s-Builder-Java:
+          CHAINCODE_NAME: basic
+          CHAINCODE_LANGUAGE: java
+          CHAINCODE_BUILDER: k8s
+
     steps:
       - template: templates/install-k8s-deps.yml
       - script: ../ci/scripts/run-k8s-test-network-basic.sh

ci/scripts/run-k8s-test-network-basic.sh

@@ -10,6 +10,7 @@ set -euo pipefail
 export CONTAINER_CLI=${CONTAINER_CLI:-docker}
 export CLIENT_LANGUAGE=${CLIENT_LANGUAGE:-typescript}
 export CHAINCODE_LANGUAGE=${CHAINCODE_LANGUAGE:-java}
+export TEST_NETWORK_CHAINCODE_BUILDER=${CHAINCODE_BUILDER:-ccaas}
 
 # Fabric version and Docker registry source: use the latest stable tag image from JFrog
 export FABRIC_VERSION=${FABRIC_VERSION:-2.4}
@@ -70,7 +71,7 @@ function createNetwork() {
   ./network channel create
 
   print "Deploying chaincode"
-  ./network chaincode deploy asset-transfer-basic basic_1.0 $TEST_NETWORK_CHAINCODE_PATH
+  ./network chaincode deploy $CHAINCODE_NAME $TEST_NETWORK_CHAINCODE_PATH
 }
 
 function stopNetwork() {

test-network-k8s/.gitignore

@@ -4,3 +4,4 @@ network-debug.log
 build/
 .env
 bin/
+*.tgz

test-network-k8s/README.md

@@ -45,7 +45,7 @@ Launch the network, create a channel, and deploy the [basic-asset-transfer](../a
 
 ./network channel create
 
-./network chaincode deploy asset-transfer-basic basic_1.0 $PWD/../asset-transfer-basic/chaincode-java
+./network chaincode deploy asset-transfer-basic ../asset-transfer-basic/chaincode-java
 ```
 
 Invoke and query chaincode:

test-network-k8s/config/org1/core.yaml

@@ -558,11 +558,18 @@ chaincode:
     # chaincode. The external builder detection processing will iterate over the
     # builders in the order specified below.
     externalBuilders:
-       - name: ccaas_builder
+      - name: ccaas_builder
-         path: /opt/hyperledger/ccaas_builder
+        path: /opt/hyperledger/ccaas_builder
-         propagateEnvironment:
+        propagateEnvironment:
-           - CHAINCODE_AS_A_SERVICE_BUILDER_CONFIG
+          - CHAINCODE_AS_A_SERVICE_BUILDER_CONFIG
 
+        # copied into the /var persistent volume share by "network up"
+      - name: k8s_builder
+        path: /var/hyperledger/fabric/external_builders/k8s_builder
+        propagateEnvironment:
+          - CORE_PEER_ID
+          - KUBERNETES_SERVICE_HOST
+          - KUBERNETES_SERVICE_PORT
 
     # The maximum duration to wait for the chaincode build and install process
     # to complete.

test-network-k8s/config/org2/core.yaml

@@ -558,11 +558,19 @@ chaincode:
     # chaincode. The external builder detection processing will iterate over the
     # builders in the order specified below.
     externalBuilders:
-       - name: ccaas_builder
+      - name: ccaas_builder
-         path: /opt/hyperledger/ccaas_builder
+        path: /opt/hyperledger/ccaas_builder
-         propagateEnvironment:
+        propagateEnvironment:
-           - CHAINCODE_AS_A_SERVICE_BUILDER_CONFIG
+          - CHAINCODE_AS_A_SERVICE_BUILDER_CONFIG
-          
+
+        # copied into the /var persistent volume share by "network up"
+      - name: k8s_builder
+        path: /var/hyperledger/fabric/external_builders/k8s_builder
+        propagateEnvironment:
+          - CORE_PEER_ID
+          - KUBERNETES_SERVICE_HOST
+          - KUBERNETES_SERVICE_PORT
+
     # The maximum duration to wait for the chaincode build and install process
     # to complete.
     installTimeout: 300s

test-network-k8s/docs/CHAINCODE_AS_A_SERVICE.md

@@ -28,7 +28,7 @@ network up
 network channel create
 ```
 ```
-network chaincode deploy    asset-transfer-basic basic_1.0 ${PWD}
+network chaincode deploy    asset-transfer-basic ${PWD}
 ```
 ```
 network chaincode metadata  asset-transfer-basic
@@ -55,7 +55,7 @@ kind load docker-image fabric-samples/asset-transfer-basic/chaincode-java
 
 ```shell
 # Assemble the chaincode package archive 
-network chaincode package basic_1.0 asset-transfer-basic $PWD/build/asset-transfer.tgz
+network chaincode package asset-transfer-basic asset-transfer-basic $PWD/build/asset-transfer.tgz
 
 # Determine the ID for the chaincode package 
 CORE_CHAINCODE_ID_NAME=$(network chaincode id $PWD/build/asset-transfer.tgz)

test-network-k8s/kube/fabric-builder-role.yaml

@@ -0,0 +1,23 @@
+#
+# Copyright IBM Corp. All Rights Reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: fabric-builder-role
+rules:
+  - apiGroups:
+      - ""
+      - apps
+    resources:
+      - pods
+      - deployments
+      - configmaps
+      - secrets
+    verbs:
+      - get
+      - watch
+      - create

test-network-k8s/kube/fabric-builder-rolebinding.yaml

@@ -0,0 +1,18 @@
+#
+# Copyright IBM Corp. All Rights Reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: fabric-builder-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: fabric-builder-role
+subjects:
+  - namespace: ${NS}
+    kind: ServiceAccount
+    name: default

test-network-k8s/kube/org1/org1-install-k8s-builder.yaml

@@ -0,0 +1,33 @@
+#
+# Copyright IBM Corp. All Rights Reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: org1-install-k8s-builder
+spec:
+  backoffLimit: 0
+  completions: 1
+  template:
+    metadata:
+      name: org1-install-k8s-builder
+    spec:
+      restartPolicy: "Never"
+      containers:
+        - name: main
+          image: ghcr.io/hyperledgendary/k8s-fabric-peer:${K8S_CHAINCODE_BUILDER_VERSION}
+          imagePullPolicy: IfNotPresent
+          command:
+            - sh
+            - -c
+            - "mkdir -p /mnt/fabric-org1/fabric/external_builders && cp -rv /opt/hyperledger/k8s_builder /mnt/fabric-org1/fabric/external_builders/"
+          volumeMounts:
+            - name: fabric-org1-volume
+              mountPath: /mnt/fabric-org1
+      volumes:
+        - name: fabric-org1-volume
+          persistentVolumeClaim:
+            claimName: fabric-org1

test-network-k8s/kube/org2/org2-install-k8s-builder.yaml

@@ -0,0 +1,33 @@
+#
+# Copyright IBM Corp. All Rights Reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: org2-install-k8s-builder
+spec:
+  backoffLimit: 0
+  completions: 1
+  template:
+    metadata:
+      name: org2-install-k8s-builder
+    spec:
+      restartPolicy: "Never"
+      containers:
+        - name: main
+          image: ghcr.io/hyperledgendary/k8s-fabric-peer:${K8S_CHAINCODE_BUILDER_VERSION}
+          imagePullPolicy: IfNotPresent
+          command:
+            - sh
+            - -c
+            - "mkdir -p /mnt/fabric-org2/fabric/external_builders && cp -rv /opt/hyperledger/k8s_builder /mnt/fabric-org2/fabric/external_builders/"
+          volumeMounts:
+            - name: fabric-org2-volume
+              mountPath: /mnt/fabric-org2
+      volumes:
+        - name: fabric-org2-volume
+          persistentVolumeClaim:
+            claimName: fabric-org2

test-network-k8s/network

@@ -30,8 +30,10 @@ export CLUSTER_NAME=${TEST_NETWORK_KIND_CLUSTER_NAME:-kind}
 export NS=${TEST_NETWORK_KUBE_NAMESPACE:-${NETWORK_NAME}}
 export DOMAIN=${TEST_NETWORK_DOMAIN:-vcap.me}
 export CHANNEL_NAME=${TEST_NETWORK_CHANNEL_NAME:-mychannel}
-export ORDERER_TIMEOUT=${TEST_NETWORK_ORDERER_TIMEOUT:-10s}     # see https://github.com/hyperledger/fabric/issues/3372
+export ORDERER_TIMEOUT=${TEST_NETWORK_ORDERER_TIMEOUT:-10s}         # see https://github.com/hyperledger/fabric/issues/3372
 export TEMP_DIR=${PWD}/build
+export CHAINCODE_BUILDER=${TEST_NETWORK_CHAINCODE_BUILDER:-ccaas}   # see https://github.com/hyperledgendary/fabric-builder-k8s/blob/main/docs/TEST_NETWORK_K8S.md
+export K8S_CHAINCODE_BUILDER_VERSION=${TEST_NETWORK_K8S_CHAINCODE_BUILDER_VERSION:-v0.4.0}
 
 LOG_FILE=${TEST_NETWORK_LOG_FILE:-network.log}
 DEBUG_FILE=${TEST_NETWORK_DEBUG_FILE:-network-debug.log}

test-network-k8s/scripts/chaincode.sh

@@ -69,40 +69,63 @@ function chaincode_command_group() {
 # Convenience routine to "do everything" required to bring up a sample CC.
 function deploy_chaincode() {
   local cc_name=$1
-  local cc_label=$2
+  local cc_label=$1
-  local cc_folder=$(absolute_path $3)
+  local cc_folder=$(absolute_path $2)
-
   local temp_folder=$(mktemp -d)
   local cc_package=${temp_folder}/${cc_name}.tgz
 
-  package_chaincode       ${cc_label} ${cc_name} ${cc_package}
+  prepare_chaincode_image ${cc_folder} ${cc_name}
+  package_chaincode       ${cc_name} ${cc_label} ${cc_package}
 
-  set_chaincode_id        ${cc_package}
+  if [ "${CHAINCODE_BUILDER}" == "ccaas" ]; then
-  set_chaincode_image     ${cc_folder}
+    set_chaincode_id      ${cc_package}
-
+    launch_chaincode      ${cc_name} ${CHAINCODE_ID} ${CHAINCODE_IMAGE}
-  build_chaincode_image   ${cc_folder} ${CHAINCODE_IMAGE}
-
-  if [ "${CLUSTER_RUNTIME}" == "kind" ]; then
-    kind_load_image       ${CHAINCODE_IMAGE}
   fi
 
-  launch_chaincode        ${cc_name} ${CHAINCODE_ID} ${CHAINCODE_IMAGE}
   activate_chaincode      ${cc_name} ${cc_package}
 }
 
-# Infer a reasonable name for the chaincode image based on the folder path conventions, or
+# Prepare a chaincode image for use in a builder package.
-# allow the user to override with TEST_NETWORK_CHAINCODE_IMAGE.
+# Sets the CHAINCODE_IMAGE environment variable
-function set_chaincode_image() {
+function prepare_chaincode_image() {
   local cc_folder=$1
+  local cc_name=$2
 
-  if [ -z "$TEST_NETWORK_CHAINCODE_IMAGE" ]; then
+  build_chaincode_image ${cc_folder} ${cc_name}
-    # cc_folder path starting with first index of "fabric-samples"
+
-    CHAINCODE_IMAGE=${cc_folder/*fabric-samples/fabric-samples}
+  if [ "${CLUSTER_RUNTIME}" == "k3s" ]; then
+    # For rancher / k3s runtimes, bypass the local container registry and load images directly from the image cache.
+    export CHAINCODE_IMAGE=${cc_name}
   else
-    CHAINCODE_IMAGE=${TEST_NETWORK_CHAINCODE_IMAGE}
+    # For KIND and k8s-builder environments, publish the image to a local docker registry
+    export CHAINCODE_IMAGE=localhost:${LOCAL_REGISTRY_PORT}/${cc_name}
+    publish_chaincode_image ${cc_name} ${CHAINCODE_IMAGE}
   fi
 }
 
+function build_chaincode_image() {
+  local cc_folder=$1
+  local cc_name=$2
+
+  push_fn "Building chaincode image ${cc_name}"
+
+  $CONTAINER_CLI build ${CONTAINER_NAMESPACE} -t ${cc_name} ${cc_folder}
+
+  pop_fn
+}
+
+# tag a docker image with a new name and publish to a remote container registry
+function publish_chaincode_image() {
+  local cc_name=$1
+  local cc_url=$2
+  push_fn "Publishing chaincode image ${cc_url}"
+
+  ${CONTAINER_CLI} tag  ${cc_name} ${cc_url}
+  ${CONTAINER_CLI} push ${cc_url}
+
+  pop_fn
+}
+
 # Convenience routine to "do everything other than package and launch" a sample CC.
 # When debugging a chaincode server, the process must be launched prior to completing
 # the chaincode lifecycle at the peer.  This routine provides a route for packaging
@@ -168,36 +191,67 @@ function invoke_chaincode() {
   sleep 2
 }
 
-function build_chaincode_image() {
-  local cc_folder=$1
-  local cc_image=$2
-
-  push_fn "Building chaincode image ${cc_image}"
-
-  $CONTAINER_CLI build ${CONTAINER_NAMESPACE} -t ${cc_image} ${cc_folder}
-
-  pop_fn
-}
-
-function kind_load_image() {
-  local cc_image=$1
-
-  push_fn "Loading chaincode to kind image plane"
-
-  kind load docker-image ${cc_image}
-
-  pop_fn
-}
-
 function package_chaincode() {
-  local cc_label=$1
+
-  local cc_name=$2
+  if [ "${CHAINCODE_BUILDER}" == "k8s" ]; then
+    package_k8s_chaincode $@
+
+  elif [ "${CHAINCODE_BUILDER}" == "ccaas" ]; then
+    package_ccaas_chaincode $@
+
+  else
+    log "Unknown CHAINCODE_BUILDER ${CHAINCODE_BUILDER}"
+    exit 1
+  fi
+}
+
+# The k8s builder expects EXACTLY an IMMUTABLE image digest referencing a SPECIFIC image layer at a container registry.
+function package_k8s_chaincode() {
+  local cc_name=$1
+  local cc_label=$2
   local cc_archive=$3
 
   local cc_folder=$(dirname $cc_archive)
   local archive_name=$(basename $cc_archive)
 
-  push_fn "Packaging chaincode ${cc_label}"
+  push_fn "Packaging k8s chaincode ${cc_archive}"
+
+  mkdir -p ${cc_folder}
+
+  # Find the docker image digest associated with the image at the container registry
+  local cc_digest=$(${CONTAINER_CLI} inspect --format='{{index .RepoDigests 0}}' ${CHAINCODE_IMAGE} | cut -d'@' -f2)
+
+  cat << IMAGEJSON-EOF > ${cc_folder}/image.json
+{
+  "name": "${CHAINCODE_IMAGE}",
+  "digest": "${cc_digest}"
+}
+IMAGEJSON-EOF
+
+  cat << METADATAJSON-EOF > ${cc_folder}/metadata.json
+{
+    "type": "k8s",
+    "label": "${cc_label}"
+}
+METADATAJSON-EOF
+
+  tar -C ${cc_folder} -zcf ${cc_folder}/code.tar.gz image.json
+  tar -C ${cc_folder} -zcf ${cc_archive} code.tar.gz metadata.json
+
+  rm ${cc_folder}/code.tar.gz
+
+  pop_fn
+}
+
+function package_ccaas_chaincode() {
+  local cc_name=$1
+  local cc_label=$2
+  local cc_archive=$3
+
+  local cc_folder=$(dirname $cc_archive)
+  local archive_name=$(basename $cc_archive)
+
+  push_fn "Packaging ccaas chaincode ${cc_label}"
 
   mkdir -p ${cc_folder}
 

test-network-k8s/scripts/fabric_config.sh

@@ -55,5 +55,23 @@ function load_org_config() {
   kubectl -n $NS create configmap org1-config --from-file=config/org1
   kubectl -n $NS create configmap org2-config --from-file=config/org2
 
+  pop_fn
+}
+
+function apply_k8s_builder_roles() {
+  push_fn "Applying k8s chaincode builder roles"
+
+  apply_template kube/fabric-builder-role.yaml
+  apply_template kube/fabric-builder-rolebinding.yaml
+
+  pop_fn
+}
+
+function apply_k8s_builders() {
+  push_fn "Installing k8s chaincode builders"
+
+  apply_template kube/org1/org1-install-k8s-builder.yaml
+  apply_template kube/org2/org2-install-k8s-builder.yaml
+
   pop_fn
 }

test-network-k8s/scripts/test_network.sh

@@ -164,6 +164,12 @@ function network_up() {
   init_storage_volumes
   load_org_config
 
+  # Service account permissions for the k8s builder
+  if [ "${CHAINCODE_BUILDER}" == "k8s" ]; then
+    apply_k8s_builder_roles
+    apply_k8s_builders
+  fi
+
   # Network TLS CAs
   init_tls_cert_issuers