test-network-k8s : Connect to Fabric services via Nginx Ingress - READY FOR MERGE (#692)

* Access the test network services via a local Nginx ingress controller. Signed-off-by: Josh Kneubuhl <jkneubuh@us.ibm.com> * Run E2E / CI test suite against the Ingress based k8s test network Signed-off-by: Josh Kneubuhl <jkneubuh@us.ibm.com> * Improved wait for Nginx Ingress - this was causing some test flakes Signed-off-by: Josh Kneubuhl <jkneubuh@us.ibm.com>
2026-06-17 15:35:09 +00:00 · 2022-03-31 12:12:51 -04:00 · 2022-03-31 12:12:51 -04:00 · a3ae179efb
commit a3ae179efb
parent 9fa171f824
26 changed files with 939 additions and 705 deletions
--- a/ci/scripts/run-k8s-test-network-basic.sh
+++ b/ci/scripts/run-k8s-test-network-basic.sh
@ -28,12 +28,12 @@ export GATEWAY_CLIENT_APPLICATION_PATH=${GATEWAY_CLIENT_APPLICATION_PATH:-../ass
 export CHANNEL_NAME=${TEST_NETWORK_CHANNEL_NAME:-mychannel}
 export CHAINCODE_NAME=${TEST_NETWORK_CHAINCODE_NAME:-asset-transfer-basic}
 export MSP_ID=${MSP_ID:-Org1MSP}
-export CRYPTO_PATH=${CRYPTO_PATH:-../../test-network-k8s/build/organizations/peerOrganizations/org1.example.com}
-export KEY_DIRECTORY_PATH=${KEY_DIRECTORY_PATH:-../../test-network-k8s/build/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore}
-export CERT_PATH=${CERT_PATH:-../../test-network-k8s/build/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/signcerts/cert.pem}
-export TLS_CERT_PATH=${TLS_CERT_PATH:-../../test-network-k8s/build/organizations/peerOrganizations/org1.example.com/msp/tlscacerts/org1-tls-ca.pem}
-export PEER_ENDPOINT=${PEER_ENDPOINT:-localhost:7051}
-export PEER_HOST_ALIAS=${PEER_HOST_ALIAS:-org1-peer1}
+export CRYPTO_PATH=${CRYPTO_PATH:-../../test-network-k8s/build/channel-msp/peerOrganizations/org1}
+export KEY_DIRECTORY_PATH=${KEY_DIRECTORY_PATH:-../../test-network-k8s/build/enrollments/org1/users/org1admin/msp/keystore}
+export CERT_PATH=${CERT_PATH:-../../test-network-k8s/build/enrollments/org1/users/org1admin/msp/signcerts/cert.pem}
+export TLS_CERT_PATH=${TLS_CERT_PATH:-../../test-network-k8s/build/channel-msp/peerOrganizations/org1/msp/tlscacerts/tlsca-signcert.pem}
+export PEER_ENDPOINT=${PEER_ENDPOINT:-org1-peer1.vcap.me:443}
+export PEER_HOST_ALIAS=${PEER_HOST_ALIAS:-org1-peer1.vcap.me}

 function print() {
  GREEN='\033[0;32m'
@ -80,29 +80,13 @@ function createNetwork() {
  ./network up
  ./network channel create

-  print "Opening gateway port-forward to 'localhost:7051'"
-  kubectl -n test-network port-forward svc/org1-peer1 7051:7051 &
-
  print "Deploying chaincode"
  ./network chaincode deploy
-
-  print "Extracting certificates"
-  kubectl \
-    -n test-network \
-    exec deploy/org1-peer1 \
-    -c main \
-    -- tar zcf - -C /var/hyperledger/fabric organizations/peerOrganizations/org1.example.com \
-    | tar zxvf - -C ../test-network-k8s/build/
 }

 function stopNetwork() {
-  pkill -f "port-forward"
-
  print "Stopping network"
  ./network down
-
-  print "Cleaning client certificates"
-  rm -rf ../test-network-k8s/build/
 }

 # Set up the suite with a KIND cluster
@ -122,8 +106,7 @@ print "Running rest-easy test"
 ( ./network rest-easy \
  && sleep 5 \
  && export SAMPLE_APIKEY='97834158-3224-4CE7-95F9-A148C886653E' \
-  && curl -s --header "X-Api-Key: ${SAMPLE_APIKEY}" "http://localhost/api/assets/asset1" | jq \
-  && curl -s --insecure --header "X-Api-Key: ${SAMPLE_APIKEY}" "https://localhost/api/assets/asset1" | jq )
+  && curl -s --header "X-Api-Key: ${SAMPLE_APIKEY}" "http://fabric-rest-sample.vcap.me/api/assets/asset1" | jq )
 print "OK"

 stopNetwork
--- a/test-network-k8s/.gitignore
+++ b/test-network-k8s/.gitignore
@ -3,3 +3,4 @@ network.log
 network-debug.log 
 build/
 .env
+bin/
--- a/test-network-k8s/config/org0/configtx.yaml
+++ b/test-network-k8s/config/org0/configtx.yaml
@ -25,7 +25,7 @@ Organizations:
    ID: OrdererMSP

    # MSPDir is the filesystem path which contains the MSP configuration
-    MSPDir: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp
+    MSPDir: ../../build/channel-msp/ordererOrganizations/org0/msp

    # Policies defines the set of policies at this level of the config tree
    # For organization policies, their canonical path is usually
@ -54,7 +54,7 @@ Organizations:
    # ID to load the MSP definition as
    ID: Org1MSP

-    MSPDir: /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/msp
+    MSPDir: ../../build/channel-msp/peerOrganizations/org1/msp

    # Policies defines the set of policies at this level of the config tree
    # For organization policies, their canonical path is usually
@ -89,7 +89,7 @@ Organizations:
    # ID to load the MSP definition as
    ID: Org2MSP

-    MSPDir: /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/msp
+    MSPDir: ../../build/channel-msp/peerOrganizations/org2/msp

    # Policies defines the set of policies at this level of the config tree
    # For organization policies, their canonical path is usually
@ -224,16 +224,16 @@ Orderer: &OrdererDefaults
    Consenters:
      - Host: org0-orderer1
        Port: 6050
-        ClientTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/tls/signcerts/cert.pem
-        ServerTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/tls/signcerts/cert.pem
+        ClientTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem
+        ServerTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem
      - Host: org0-orderer2
        Port: 6050
-        ClientTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/tls/signcerts/cert.pem
-        ServerTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/tls/signcerts/cert.pem
+        ClientTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer2/tls/signcerts/tls-cert.pem
+        ServerTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer2/tls/signcerts/tls-cert.pem
      - Host: org0-orderer3
        Port: 6050
-        ClientTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/tls/signcerts/cert.pem
-        ServerTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/tls/signcerts/cert.pem
+        ClientTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer3/tls/signcerts/tls-cert.pem
+        ServerTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer3/tls/signcerts/tls-cert.pem


    # Options to be specified for all the etcd/raft nodes. The values here
--- a/test-network-k8s/kube/fabric-rest-sample.yaml
+++ b/test-network-k8s/kube/fabric-rest-sample.yaml
@ -236,23 +236,26 @@ spec:
  selector:
    app: fabric-rest-sample

-
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+  labels:
+    app: fabric-rest-sample
  name: fabric-rest-sample
-#  annotations:
-#    nginx.ingress.kubernetes.io/rewrite-target: /$1
 spec:
+  ingressClassName: nginx
  rules:
-    - http:
+    - host: fabric-rest-sample.${DOMAIN}
+      http:
        paths:
-#          - path: "/fabric-rest-sample/(.*)"
-          - path: "/"
-            pathType: Prefix
-            backend:
+          - backend:
              service:
                name: fabric-rest-sample
                port:
-                  number: 3000
+                  name: http
+            path: /
+            pathType: ImplementationSpecific
+
--- a/test-network-k8s/kube/org0/org0-admin-cli.yaml
+++ b/test-network-k8s/kube/org0/org0-admin-cli.yaml
@ -1,61 +0,0 @@
-#
-# Copyright IBM Corp. All Rights Reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: org0-admin-cli
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: org0-admin-cli
-  template:
-    metadata:
-      labels:
-        app: org0-admin-cli
-    spec:
-      containers:
-        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-tools:{{FABRIC_VERSION}}
-          imagePullPolicy: IfNotPresent
-          env:
-            - name: FABRIC_CFG_PATH
-              value: /var/hyperledger/fabric/config
-          args:
-            - sleep
-            - "2147483647"
-          workingDir: /root
-          volumeMounts:
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-            - name: fabric-config
-              mountPath: /var/hyperledger/fabric/config
-
-      # This init container will unfurl all of the MSP archives listed in the msp-config config map.
-      initContainers:
-        - name: msp-unfurl
-          image: busybox
-          command:
-            - sh
-            - -c
-            - "for msp in $(ls /msp/msp-*.tgz); do echo $msp && tar zxvf $msp -C /var/hyperledger/fabric; done"
-          volumeMounts:
-            - name: msp-config
-              mountPath: /msp
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-
-      volumes:
-        - name: fabric-volume
-          persistentVolumeClaim:
-            claimName: fabric-org0
-        - name: fabric-config
-          configMap:
-            name: org0-config
-        - name: msp-config
-          configMap:
-            name: msp-config
--- a/test-network-k8s/kube/org0/org0-ca.yaml
+++ b/test-network-k8s/kube/org0/org0-ca.yaml
@ -17,6 +17,7 @@ spec:
    - localhost
    - org0-ca
    - org0-ca.test-network.svc.cluster.local
+    - org0-ca.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org0-ca-tls-cert
@ -40,7 +41,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-ca:{{FABRIC_CA_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-ca:${FABRIC_CA_VERSION}
          imagePullPolicy: IfNotPresent
          env:
            - name: FABRIC_CA_SERVER_CA_NAME
@ -89,8 +90,35 @@ metadata:
  name: org0-ca
 spec:
  ports:
-    - name: tls
+    - name: https
      port: 443
      protocol: TCP
  selector:
    app: org0-ca
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org0-ca
+  name: org0-ca
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org0-ca.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-ca
+                port:
+                  name: https
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+      - org0-ca.${DOMAIN}
--- a/test-network-k8s/kube/org0/org0-orderer1.yaml
+++ b/test-network-k8s/kube/org0/org0-orderer1.yaml
@ -9,7 +9,7 @@ apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
  name: org0-orderer1-tls-cert
-  namespace: test-network
+  namespace: ${NS}
 spec:
  isCA: false
  privateKey:
@ -18,7 +18,9 @@ spec:
  dnsNames:
    - localhost
    - org0-orderer1
-    - org0-orderer1.test-network.svc.cluster.local
+    - org0-orderer1.${NS}.svc.cluster.local
+    - org0-orderer1.${DOMAIN}
+    - org0-orderer1-admin.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org0-orderer1-tls-cert
@ -42,6 +44,12 @@ data:
  ORDERER_GENERAL_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
  ORDERER_GENERAL_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
  ORDERER_GENERAL_BOOTSTRAPMETHOD: none
+  ORDERER_ADMIN_TLS_ENABLED: "true"
+  ORDERER_ADMIN_TLS_CERTIFICATE: /var/hyperledger/fabric/config/tls/tls.crt
+  ORDERER_ADMIN_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
+  ORDERER_ADMIN_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
+  # Authenticate client connections with the org's ecert / admin user enrollments
+  ORDERER_ADMIN_TLS_CLIENTROOTCAS: "[/var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp/cacerts/org0-ca.pem]"
  ORDERER_FILELEDGER_LOCATION: /var/hyperledger/fabric/data/orderer1
  ORDERER_CONSENSUS_WALDIR: /var/hyperledger/fabric/data/orderer1/etcdraft/wal
  ORDERER_CONSENSUS_SNAPDIR: /var/hyperledger/fabric/data/orderer1/etcdraft/wal
@ -65,7 +73,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-orderer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-orderer:${FABRIC_VERSION}
          imagePullPolicy: IfNotPresent
          envFrom:
            - configMapRef:
@ -110,3 +118,42 @@ spec:
      protocol: TCP
  selector:
    app: org0-orderer1
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org0-orderer1
+  name: org0-orderer1
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org0-orderer1.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer1
+                port:
+                  name: general
+            path: /
+            pathType: ImplementationSpecific
+    - host: org0-orderer1-admin.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer1
+                port:
+                  name: admin
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org0-orderer1.${DOMAIN}
+    - hosts:
+        - org0-orderer1-admin.${DOMAIN}
--- a/test-network-k8s/kube/org0/org0-orderer2.yaml
+++ b/test-network-k8s/kube/org0/org0-orderer2.yaml
@ -3,12 +3,13 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 #
+
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
  name: org0-orderer2-tls-cert
-  namespace: test-network
+  namespace: ${NS}
 spec:
  isCA: false
  privateKey:
@ -17,7 +18,9 @@ spec:
  dnsNames:
    - localhost
    - org0-orderer2
-    - org0-orderer2.test-network.svc.cluster.local
+    - org0-orderer2.${NS}.svc.cluster.local
+    - org0-orderer2.${DOMAIN}
+    - org0-orderer2-admin.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org0-orderer2-tls-cert
@ -41,6 +44,12 @@ data:
  ORDERER_GENERAL_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
  ORDERER_GENERAL_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
  ORDERER_GENERAL_BOOTSTRAPMETHOD: none
+  ORDERER_ADMIN_TLS_ENABLED: "true"
+  ORDERER_ADMIN_TLS_CERTIFICATE: /var/hyperledger/fabric/config/tls/tls.crt
+  ORDERER_ADMIN_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
+  ORDERER_ADMIN_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
+  # Authenticate client connections with the org's ecert / admin user enrollments
+  ORDERER_ADMIN_TLS_CLIENTROOTCAS: "[/var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/msp/cacerts/org0-ca.pem]"
  ORDERER_FILELEDGER_LOCATION: /var/hyperledger/fabric/data/orderer2
  ORDERER_CONSENSUS_WALDIR: /var/hyperledger/fabric/data/orderer2/etcdraft/wal
  ORDERER_CONSENSUS_SNAPDIR: /var/hyperledger/fabric/data/orderer2/etcdraft/wal
@ -64,7 +73,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-orderer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-orderer:${FABRIC_VERSION}
          imagePullPolicy: IfNotPresent
          envFrom:
            - configMapRef:
@ -109,3 +118,42 @@ spec:
      protocol: TCP
  selector:
    app: org0-orderer2
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org0-orderer2
+  name: org0-orderer2
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org0-orderer2.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer2
+                port:
+                  name: general
+            path: /
+            pathType: ImplementationSpecific
+    - host: org0-orderer2-admin.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer2
+                port:
+                  name: admin
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org0-orderer2.${DOMAIN}
+    - hosts:
+        - org0-orderer2-admin.${DOMAIN}
--- a/test-network-k8s/kube/org0/org0-orderer3.yaml
+++ b/test-network-k8s/kube/org0/org0-orderer3.yaml
@ -3,12 +3,13 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 #
+
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
  name: org0-orderer3-tls-cert
-  namespace: test-network
+  namespace: ${NS}
 spec:
  isCA: false
  privateKey:
@ -17,7 +18,9 @@ spec:
  dnsNames:
    - localhost
    - org0-orderer3
-    - org0-orderer3.test-network.svc.cluster.local
+    - org0-orderer3.${NS}.svc.cluster.local
+    - org0-orderer3.${DOMAIN}
+    - org0-orderer3-admin.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org0-orderer3-tls-cert
@ -41,6 +44,12 @@ data:
  ORDERER_GENERAL_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
  ORDERER_GENERAL_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
  ORDERER_GENERAL_BOOTSTRAPMETHOD: none
+  ORDERER_ADMIN_TLS_ENABLED: "true"
+  ORDERER_ADMIN_TLS_CERTIFICATE: /var/hyperledger/fabric/config/tls/tls.crt
+  ORDERER_ADMIN_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
+  ORDERER_ADMIN_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
+  # Authenticate client connections with the org's ecert / admin user enrollments
+  ORDERER_ADMIN_TLS_CLIENTROOTCAS: "[/var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/msp/cacerts/org0-ca.pem]"
  ORDERER_FILELEDGER_LOCATION: /var/hyperledger/fabric/data/orderer3
  ORDERER_CONSENSUS_WALDIR: /var/hyperledger/fabric/data/orderer3/etcdraft/wal
  ORDERER_CONSENSUS_SNAPDIR: /var/hyperledger/fabric/data/orderer3/etcdraft/wal
@ -64,7 +73,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-orderer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-orderer:${FABRIC_VERSION}
          imagePullPolicy: IfNotPresent
          envFrom:
            - configMapRef:
@ -109,3 +118,42 @@ spec:
      protocol: TCP
  selector:
    app: org0-orderer3
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org0-orderer3
+  name: org0-orderer3
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org0-orderer3.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer3
+                port:
+                  name: general
+            path: /
+            pathType: ImplementationSpecific
+    - host: org0-orderer3-admin.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer3
+                port:
+                  name: admin
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org0-orderer3.${DOMAIN}
+    - hosts:
+        - org0-orderer3-admin.${DOMAIN}
--- a/test-network-k8s/kube/org1/org1-admin-cli.yaml
+++ b/test-network-k8s/kube/org1/org1-admin-cli.yaml
@ -1,65 +0,0 @@
-#
-# Copyright IBM Corp. All Rights Reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: org1-admin-cli
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: org1-admin-cli
-  template:
-    metadata:
-      labels:
-        app: org1-admin-cli
-    spec:
-      containers:
-        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-tools:{{FABRIC_VERSION}}
-          imagePullPolicy: IfNotPresent
-          env:
-            - name: FABRIC_CFG_PATH
-              value: /var/hyperledger/fabric/config
-            - name: CORE_PEER_MSPCONFIGPATH
-              value: /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp
-            - name: CORE_PEER_TLS_ROOTCERT_FILE
-              value: /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/msp/tlscacerts/org1-tls-ca.pem
-          args:
-            - sleep
-            - "2147483647"
-          workingDir: /root
-          volumeMounts:
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-            - name: fabric-config
-              mountPath: /var/hyperledger/fabric/config
-
-      # This init container will unfurl all of the MSP archives listed in the msp-config config map.
-      initContainers:
-        - name: msp-unfurl
-          image: busybox
-          command:
-            - sh
-            - -c
-            - "for msp in $(ls /msp/msp-*.tgz); do echo $msp && tar zxvf $msp -C /var/hyperledger/fabric; done"
-          volumeMounts:
-            - name: msp-config
-              mountPath: /msp
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-
-      volumes:
-        - name: fabric-volume
-          persistentVolumeClaim:
-            claimName: fabric-org1
-        - name: fabric-config
-          configMap:
-            name: org1-config
-        - name: msp-config
-          configMap:
-            name: msp-config
--- a/test-network-k8s/kube/org1/org1-ca.yaml
+++ b/test-network-k8s/kube/org1/org1-ca.yaml
@ -17,6 +17,7 @@ spec:
    - localhost
    - org1-ca
    - org1-ca.test-network.svc.cluster.local
+    - org1-ca.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org1-ca-tls-cert
@ -40,7 +41,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-ca:{{FABRIC_CA_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-ca:${FABRIC_CA_VERSION}
          imagePullPolicy: IfNotPresent
          env:
            - name: FABRIC_CA_SERVER_CA_NAME
@ -88,8 +89,36 @@ metadata:
  name: org1-ca
 spec:
  ports:
-    - name: tls
+    - name: https
      port: 443
      protocol: TCP
  selector:
    app: org1-ca
+    
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org1-ca
+  name: org1-ca
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org1-ca.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org1-ca
+                port:
+                  name: https
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org1-ca.${DOMAIN}
+  
--- a/test-network-k8s/kube/org1/org1-peer1.yaml
+++ b/test-network-k8s/kube/org1/org1-peer1.yaml
@ -17,8 +17,10 @@ spec:
  dnsNames:
    - localhost
    - org1-peer1
-    - org1-peer-gateway-svc
    - org1-peer1.test-network.svc.cluster.local
+    - org1-peer1.${DOMAIN}
+    - org1-peer-gateway-svc
+    - org1-peer-gateway-svc.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org1-peer1-tls-cert
@ -74,7 +76,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-peer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-peer:${FABRIC_VERSION}
          imagePullPolicy: IfNotPresent
          envFrom:
            - configMapRef:
@ -118,7 +120,7 @@ metadata:
  name: org1-peer1
 spec:
  ports:
-    - name: gossip
+    - name: grpc
      port: 7051
      protocol: TCP
    - name: chaincode
@ -136,8 +138,46 @@ metadata:
  name: org1-peer-gateway-svc
 spec:
  ports:
-    - name: gossip
+    - name: grpc
      port: 7051
      protocol: TCP
  selector:
    org: org1
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org1-peer1
+  name: org1-peer1
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org1-peer1.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org1-peer1
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+    - host: org1-peer-gateway-svc.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org1-peer1
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org1-peer1.${DOMAIN}
+    - hosts:
+        - org1-peer-gateway-svc.${DOMAIN}
--- a/test-network-k8s/kube/org1/org1-peer2.yaml
+++ b/test-network-k8s/kube/org1/org1-peer2.yaml
@ -19,6 +19,7 @@ spec:
    - org1-peer2
    - org1-peer-gateway-svc
    - org1-peer2.test-network.svc.cluster.local
+    - org1-peer2.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org1-peer2-tls-cert
@ -75,7 +76,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-peer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-peer:${FABRIC_VERSION}
          imagePullPolicy: IfNotPresent
          envFrom:
            - configMapRef:
@ -119,7 +120,7 @@ metadata:
  name: org1-peer2
 spec:
  ports:
-    - name: gossip
+    - name: grpc
      port: 7051
      protocol: TCP
    - name: chaincode
@ -130,3 +131,29 @@ spec:
      protocol: TCP
  selector:
    app: org1-peer2
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org1-peer2
+  name: org1-peer2
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org1-peer2.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org1-peer2
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org1-peer2.${DOMAIN}
--- a/test-network-k8s/kube/org2/org2-admin-cli.yaml
+++ b/test-network-k8s/kube/org2/org2-admin-cli.yaml
@ -1,65 +0,0 @@
-#
-# Copyright IBM Corp. All Rights Reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: org2-admin-cli
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: org2-admin-cli
-  template:
-    metadata:
-      labels:
-        app: org2-admin-cli
-    spec:
-      containers:
-        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-tools:{{FABRIC_VERSION}}
-          imagePullPolicy: IfNotPresent
-          env:
-            - name: FABRIC_CFG_PATH
-              value: /var/hyperledger/fabric/config
-            - name: CORE_PEER_MSPCONFIGPATH
-              value: /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp
-            - name: CORE_PEER_TLS_ROOTCERT_FILE
-              value: /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/msp/tlscacerts/org2-tls-ca.pem
-          args:
-            - sleep
-            - "2147483647"
-          workingDir: /root
-          volumeMounts:
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-            - name: fabric-config
-              mountPath: /var/hyperledger/fabric/config
-
-      # This init container will unfurl all of the MSP archives listed in the msp-config config map.
-      initContainers:
-        - name: msp-unfurl
-          image: busybox
-          command:
-            - sh
-            - -c
-            - "for msp in $(ls /msp/msp-*.tgz); do echo $msp && tar zxvf $msp -C /var/hyperledger/fabric; done"
-          volumeMounts:
-            - name: msp-config
-              mountPath: /msp
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-
-      volumes:
-        - name: fabric-volume
-          persistentVolumeClaim:
-            claimName: fabric-org2
-        - name: fabric-config
-          configMap:
-            name: org2-config
-        - name: msp-config
-          configMap:
-            name: msp-config
--- a/test-network-k8s/kube/org2/org2-ca.yaml
+++ b/test-network-k8s/kube/org2/org2-ca.yaml
@ -17,6 +17,7 @@ spec:
    - localhost
    - org2-ca
    - org2-ca.test-network.svc.cluster.local
+    - org2-ca.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org2-ca-tls-cert
@ -40,7 +41,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-ca:{{FABRIC_CA_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-ca:${FABRIC_CA_VERSION}
          imagePullPolicy: IfNotPresent
          env:
            - name: FABRIC_CA_SERVER_CA_NAME
@ -88,8 +89,36 @@ metadata:
  name: org2-ca
 spec:
  ports:
-    - name: tls
+    - name: https
      port: 443
      protocol: TCP
  selector:
    app: org2-ca
+    
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org2-ca
+  name: org2-ca
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org2-ca.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org2-ca
+                port:
+                  name: https
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org2-ca.${DOMAIN}
+    
--- a/test-network-k8s/kube/org2/org2-peer1.yaml
+++ b/test-network-k8s/kube/org2/org2-peer1.yaml
@ -17,8 +17,10 @@ spec:
  dnsNames:
    - localhost
    - org2-peer1
-    - org2-peer-gateway-svc
    - org2-peer1.test-network.svc.cluster.local
+    - org2-peer1.${DOMAIN}
+    - org2-peer-gateway-svc
+    - org2-peer-gateway-svc.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org2-peer1-tls-cert
@ -74,7 +76,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-peer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-peer:${FABRIC_VERSION}
          imagePullPolicy: IfNotPresent
          envFrom:
            - configMapRef:
@ -118,7 +120,7 @@ metadata:
  name: org2-peer1
 spec:
  ports:
-    - name: gossip
+    - name: grpc
      port: 7051
      protocol: TCP
    - name: chaincode
@ -136,8 +138,46 @@ metadata:
  name: org2-peer-gateway-svc
 spec:
  ports:
-    - name: gossip
+    - name: grpc
      port: 7051
      protocol: TCP
  selector:
    org: org2
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org2-peer1
+  name: org2-peer1
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org2-peer1.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org2-peer1
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+    - host: org2-peer-gateway-svc.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org2-peer1
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org2-peer1.${DOMAIN}
+    - hosts:
+        - org2-peer-gateway-svc.${DOMAIN}
--- a/test-network-k8s/kube/org2/org2-peer2.yaml
+++ b/test-network-k8s/kube/org2/org2-peer2.yaml
@ -19,6 +19,7 @@ spec:
    - org2-peer2
    - org2-peer-gateway-svc
    - org2-peer2.test-network.svc.cluster.local
+    - org2-peer2.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org2-peer2-tls-cert
@ -74,7 +75,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-peer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-peer:${FABRIC_VERSION}
          imagePullPolicy: IfNotPresent
          envFrom:
            - configMapRef:
@ -118,7 +119,7 @@ metadata:
  name: org2-peer2
 spec:
  ports:
-    - name: gossip
+    - name: grpc
      port: 7051
      protocol: TCP
    - name: chaincode
@ -129,3 +130,29 @@ spec:
      protocol: TCP
  selector:
    app: org2-peer2
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org2-peer2
+  name: org2-peer2
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org2-peer2.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org2-peer2
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org2-peer2.${DOMAIN}
--- a/test-network-k8s/network
+++ b/test-network-k8s/network
@ -6,28 +6,25 @@
 #
 set -o errexit

-# todo: better handling for input parameters.
+# todo: better handling for input parameters.  Argbash?
 # todo: skip storage volume init if deploying to a remote / cloud cluster (ICP IKS ROKS etc...)
 # todo: for logging, set up a stack and allow multi-line status output codes
-# todo: refactor - lots of for-org-in-0-to-2-...
-# todo: find a better technique for passing input commands to a remote kube exec
-# todo: register tls csr.hosts w/ kube DNS domain .NS.svc.cluster.local
 # todo: user:pass auth for tls and ecert bootstrap admins.  here and in the server-config.yaml
-# todo: set tls.certfiles= ... arg in deployment env / yaml
 # todo: refactor chaincode install to support other chaincode routines
-# todo: consider using templates for boilerplate network nodes (orderers, peers, ...)
 # todo: allow the user to specify the chaincode name (hardcoded as 'basic') both in install and invoke/query
 # todo: track down a nasty bug whereby the CA service endpoints (kube services) will occasionally reject TCP connections after network down/up.  This is patched by introducing a 10s sleep after the deployments are up...
-# todo: refactor query/invoke to specify chaincode name (-n param)

-CONTAINER_CLI=${CONTAINER_CLI:-docker}
-FABRIC_VERSION=${TEST_NETWORK_FABRIC_VERSION:-2.4}
-FABRIC_CA_VERSION=${TEST_NETWORK_FABRIC_CA_VERSION:-1.5}
-FABRIC_CONTAINER_REGISTRY=${TEST_NETWORK_FABRIC_CONTAINER_REGISTRY:-hyperledger}
-NETWORK_NAME=${TEST_NETWORK_NAME:-test-network}
-CLUSTER_NAME=${TEST_NETWORK_KIND_CLUSTER_NAME:-kind}
-NS=${TEST_NETWORK_KUBE_NAMESPACE:-${NETWORK_NAME}}
-CHANNEL_NAME=${TEST_NETWORK_CHANNEL_NAME:-mychannel}
+export CONTAINER_CLI=${CONTAINER_CLI:-docker}
+export FABRIC_VERSION=${TEST_NETWORK_FABRIC_VERSION:-2.4.3}
+export FABRIC_CA_VERSION=${TEST_NETWORK_FABRIC_CA_VERSION:-1.5.2}
+export FABRIC_CONTAINER_REGISTRY=${TEST_NETWORK_FABRIC_CONTAINER_REGISTRY:-hyperledger}
+export NETWORK_NAME=${TEST_NETWORK_NAME:-test-network}
+export CLUSTER_NAME=${TEST_NETWORK_KIND_CLUSTER_NAME:-kind}
+export NS=${TEST_NETWORK_KUBE_NAMESPACE:-${NETWORK_NAME}}
+export DOMAIN=${TEST_NETWORK_DOMAIN:-vcap.me}
+export CHANNEL_NAME=${TEST_NETWORK_CHANNEL_NAME:-mychannel}
+export TEMP_DIR=${PWD}/build
+
 LOG_FILE=${TEST_NETWORK_LOG_FILE:-network.log}
 DEBUG_FILE=${TEST_NETWORK_DEBUG_FILE:-network-debug.log}
 LOCAL_REGISTRY_NAME=${TEST_NETWORK_LOCAL_REGISTRY_NAME:-kind-registry}
@ -50,6 +47,7 @@ function print_help() {
  log "Fabric CA Version    \t: ${FABRIC_CA_VERSION}"
  log "Container Registry   \t: ${FABRIC_CONTAINER_REGISTRY}"
  log "Network name       \t\t: ${NETWORK_NAME}"
+  log "Ingress domain     \t\t: ${DOMAIN}"
  log "Channel name       \t\t: ${CHANNEL_NAME}"
  log
  log "--- Chaincode Information"
@ -71,8 +69,6 @@ function print_help() {
  log "Debug log file     \t\t: ${DEBUG_FILE}"
  log

-
-
  echo todo: help output, parse mode, flags, env, etc.
 }

--- a/test-network-k8s/scripts/chaincode.sh
+++ b/test-network-k8s/scripts/chaincode.sh
@ -22,37 +22,23 @@ function package_chaincode_for() {
  pop_fn
 }

-# Copy the chaincode archive from the local host to the org admin
-function transfer_chaincode_archive_for() {
-  local org=$1
-  local cc_archive="build/chaincode/${CHAINCODE_NAME}.tgz"
-  push_fn "Transferring chaincode archive to ${org}"
-
-  # Like kubectl cp, but targeted to a deployment rather than an individual pod.
-  tar cf - ${cc_archive} | kubectl -n $NS exec -i deploy/${org}-admin-cli -c main -- tar xvf -
-
-  pop_fn
-}
-
 function install_chaincode_for() {
  local org=$1
  local peer=$2
-  push_fn "Installing chaincode for org ${org}  peer ${peer}"
+  push_fn "Installing chaincode for org ${org} peer ${peer}"

-  # Install the chaincode
-  echo 'set -x
-  export CORE_PEER_ADDRESS='${org}'-'${peer}':7051
-  peer lifecycle chaincode install build/chaincode/'${CHAINCODE_NAME}'.tgz
-  ' | exec kubectl -n $NS exec deploy/${org}-admin-cli -c main -i -- /bin/bash
+  export_peer_context $org $peer
+
+  peer lifecycle chaincode install build/chaincode/${CHAINCODE_NAME}.tgz

  pop_fn
 }

 function launch_chaincode_service() {
  local org=$1
-  local cc_id=$2
-  local cc_image=$3
-  local peer=$4
+  local peer=$2
+  local cc_id=$3
+  local cc_image=$4
  push_fn "Launching chaincode container \"${cc_image}\""

  # The chaincode endpoint needs to have the generated chaincode ID available in the environment.
@ -72,76 +58,72 @@ function launch_chaincode_service() {

 function activate_chaincode_for() {
  local org=$1
-  local cc_id=$2
-  push_fn "Activating chaincode ${CHAINCODE_ID}"
+  local peer=$2
+  local cc_id=$3
+  push_fn "Activating $org chaincode ${CHAINCODE_ID}"

-  echo 'set -x 
-  export CORE_PEER_ADDRESS='${org}'-peer1:7051
+  export_peer_context $org $peer

  peer lifecycle \
    chaincode approveformyorg \
-    --channelID '${CHANNEL_NAME}' \
-    --name '${CHAINCODE_NAME}' \
-    --version 1 \
-    --package-id '${cc_id}' \
-    --sequence 1 \
-    -o org0-orderer1:6050 \
-    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
+    --channelID     ${CHANNEL_NAME} \
+    --name          ${CHAINCODE_NAME} \
+    --version       1 \
+    --package-id    ${cc_id} \
+    --sequence      1 \
+    --orderer       org0-orderer1.${DOMAIN}:443 \
+    --tls --cafile  ${TEMP_DIR}/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem

  peer lifecycle \
    chaincode commit \
-    --channelID '${CHANNEL_NAME}' \
-    --name '${CHAINCODE_NAME}' \
-    --version 1 \
-    --sequence 1 \
-    -o org0-orderer1:6050 \
-    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
-  ' | exec kubectl -n $NS exec deploy/${org}-admin-cli -c main -i -- /bin/bash
+    --channelID     ${CHANNEL_NAME} \
+    --name          ${CHAINCODE_NAME} \
+    --version       1 \
+    --sequence      1 \
+    --orderer       org0-orderer1.${DOMAIN}:443 \
+    --tls --cafile  ${TEMP_DIR}/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem

  pop_fn
 }

 function query_chaincode() {
  set -x
-  # todo: mangle additional $@ parameters with bash escape quotations
-  echo '
-  export CORE_PEER_ADDRESS=org1-peer1:7051
-  peer chaincode query -n '${CHAINCODE_NAME}' -C '${CHANNEL_NAME}' -c '"'$@'"'
-  ' | exec kubectl -n $NS exec deploy/org1-admin-cli -c main -i -- /bin/bash
+
+  export_peer_context org1 peer1
+
+  peer chaincode query \
+    -n  $CHAINCODE_NAME \
+    -C  $CHANNEL_NAME \
+    -c  $@
 }

 function query_chaincode_metadata() {
  set -x
  local args='{"Args":["org.hyperledger.fabric:GetMetadata"]}'
-  # todo: mangle additional $@ parameters with bash escape quotations
+
+  log ''
  log 'Org1-Peer1:'
-  echo '
-  export CORE_PEER_ADDRESS=org1-peer1:7051
-  peer chaincode query -n '${CHAINCODE_NAME}' -C '${CHANNEL_NAME}' -c '"'$args'"'
-  ' | exec kubectl -n $NS exec deploy/org1-admin-cli -c main -i -- /bin/bash
+  export_peer_context org1 peer1
+  peer chaincode query -n $CHAINCODE_NAME -C $CHANNEL_NAME -c $args

  log ''
  log 'Org1-Peer2:'
-  echo '
-  export CORE_PEER_ADDRESS=org1-peer2:7051
-  peer chaincode query -n '${CHAINCODE_NAME}' -C '${CHANNEL_NAME}' -c '"'$args'"'
-  ' | exec kubectl -n $NS exec deploy/org1-admin-cli -c main -i -- /bin/bash
-
+  export_peer_context org1 peer2
+  peer chaincode query -n $CHAINCODE_NAME -C $CHANNEL_NAME -c $args
 }

 function invoke_chaincode() {
  # set -x
  # todo: mangle additional $@ parameters with bash escape quotations
-  echo '
-  export CORE_PEER_ADDRESS=org1-peer1:7051
-  peer chaincode \
-    invoke \
-    -o org0-orderer1:6050 \
-    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem \
-    -n '${CHAINCODE_NAME}' \
-    -C '${CHANNEL_NAME}' \
-    -c '"'$@'"'
-  ' | exec kubectl -n $NS exec deploy/org1-admin-cli -c main -i -- /bin/bash
+
+  export_peer_context org1 peer1
+
+  peer chaincode invoke \
+    -n              $CHAINCODE_NAME \
+    -C              $CHANNEL_NAME \
+    -c              $@ \
+    --orderer       org0-orderer1.${DOMAIN}:443 \
+    --tls --cafile  ${TEMP_DIR}/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem

  sleep 2
 }
@ -162,7 +144,7 @@ function install_chaincode() {
  local org=org1

  package_chaincode_for ${org}
-  transfer_chaincode_archive_for ${org}
+
  install_chaincode_for ${org} peer1
  install_chaincode_for ${org} peer2

@ -174,7 +156,11 @@ function activate_chaincode() {
  set -x

  set_chaincode_id
-  activate_chaincode_for org1 $CHAINCODE_ID
+  activate_chaincode_for org1 peer1 $CHAINCODE_ID
+
+  # jdk: does activation on a single peer apply to all peers in the org?  This is an error:
+#  activate_chaincode_for org1 peer1 $CHAINCODE_ID
+
 }

 # Install, launch, and activate the chaincode
@ -182,8 +168,8 @@ function deploy_chaincode() {
  set -x

  install_chaincode
-  launch_chaincode_service org1 $CHAINCODE_ID $CHAINCODE_IMAGE peer1
-  launch_chaincode_service org1 $CHAINCODE_ID $CHAINCODE_IMAGE peer2
+  launch_chaincode_service org1 peer1 $CHAINCODE_ID $CHAINCODE_IMAGE
+  launch_chaincode_service org1 peer2 $CHAINCODE_ID $CHAINCODE_IMAGE
  activate_chaincode
 }

--- a/test-network-k8s/scripts/channel.sh
+++ b/test-network-k8s/scripts/channel.sh
@ -5,39 +5,140 @@
 # SPDX-License-Identifier: Apache-2.0
 #

-function create_channel_org_MSP() {
+function channel_up() {
+  set -x
+
+  register_org_admins
+  enroll_org_admins
+
+  create_channel_MSP
+  create_genesis_block
+
+  join_channel_orderers
+  join_channel_peers
+}
+
+function register_org_admins() {
+  push_fn "Registering org Admin users"
+
+  register_org_admin org0 org0admin org0adminpw
+  register_org_admin org1 org1admin org1adminpw
+  register_org_admin org2 org2admin org2adminpw
+
+  pop_fn
+}
+
+# Register the org admin user
+function register_org_admin() {
+  local type=admin
  local org=$1
-  local org_type=$2
-  local ecert_ca=${org}-ca
+  local id_name=$2
+  local id_secret=$3
+  local ca_name=${org}-ca

-  echo 'set -x
+  echo "Registering org admin $username"

-  mkdir -p /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/cacerts
-  cp \
-    $FABRIC_CA_CLIENT_HOME/'${ecert_ca}'/rcaadmin/msp/cacerts/'${ecert_ca}'.pem \
-    /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/cacerts
+  cat <<EOF | kubectl -n $NS exec deploy/${ca_name} -i -- /bin/sh

-  mkdir -p /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/tlscacerts
-  cp \
-    /var/hyperledger/fabric/config/tls/ca.crt \
-    /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/tlscacerts/'${org}'-tls-ca.pem
+  set -x
+  export FABRIC_CA_CLIENT_HOME=/var/hyperledger/fabric-ca-client
+  export FABRIC_CA_CLIENT_TLS_CERTFILES=/var/hyperledger/fabric/config/tls/ca.crt

-  echo "NodeOUs:
-    Enable: true
-    ClientOUIdentifier:
-      Certificate: cacerts/'${ecert_ca}'.pem
-      OrganizationalUnitIdentifier: client
-    PeerOUIdentifier:
-      Certificate: cacerts/'${ecert_ca}'.pem
-      OrganizationalUnitIdentifier: peer
-    AdminOUIdentifier:
-      Certificate: cacerts/'${ecert_ca}'.pem
-      OrganizationalUnitIdentifier: admin
-    OrdererOUIdentifier:
-      Certificate: cacerts/'${ecert_ca}'.pem
-      OrganizationalUnitIdentifier: orderer "> /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/config.yaml
+  fabric-ca-client register \
+    --id.name   ${id_name} \
+    --id.secret ${id_secret} \
+    --id.type   ${type} \
+    --url       https://${ca_name} \
+    --mspdir    /var/hyperledger/fabric-ca-client/${ca_name}/rcaadmin/msp \
+    --id.attrs  "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
+EOF
+}

-  ' | exec kubectl -n $NS exec deploy/${ecert_ca} -i -- /bin/sh
+function enroll_org_admins() {
+  push_fn "Enrolling org Admin users"
+
+  enroll_org_admin orderer  org0 org0admin org0adminpw
+  enroll_org_admin peer     org1 org1admin org1adminpw
+  enroll_org_admin peer     org2 org2admin org2adminpw
+
+  pop_fn
+}
+
+# Enroll the admin client to the local certificate storage folder.
+function enroll_org_admin() {
+  local type=$1
+  local org=$2
+  local username=$3
+  local password=$4
+
+  echo "Enrolling $type org admin $username"
+
+  ENROLLMENTS_DIR=${TEMP_DIR}/enrollments
+  ORG_ADMIN_DIR=${ENROLLMENTS_DIR}/${org}/users/${username}
+
+  # skip the enrollment if the admin certificate is available.
+  if [ -f "${ORG_ADMIN_DIR}/msp/keystore/key.pem" ]; then
+    echo "Found an existing admin enrollment at ${ORG_ADMIN_DIR}"
+    return
+  fi
+
+  # Determine the CA information and TLS certificate
+  CA_NAME=${org}-ca
+  CA_DIR=${TEMP_DIR}/cas/${CA_NAME}
+  mkdir -p ${CA_DIR}
+
+  CA_AUTH=${username}:${password}
+  CA_HOST=${CA_NAME}.${DOMAIN}
+  CA_PORT=443
+  CA_URL=https://${CA_AUTH}@${CA_HOST}:${CA_PORT}
+
+  # Read the CA's TLS certificate from the cert-manager CA secret
+  echo "retrieving ${org}-ca TLS cert"
+  kubectl -n $NS get secret ${org}-ca-tls-cert -o json \
+    | jq -r .data.\"tls.crt\" \
+    | base64 -d \
+    > ${CA_DIR}/tls-cert.pem
+
+  # enroll the org admin
+  FABRIC_CA_CLIENT_HOME=${ORG_ADMIN_DIR} fabric-ca-client enroll \
+    --url ${CA_URL} \
+    --tls.certfiles ${CA_DIR}/tls-cert.pem
+
+  # Construct an msp config.yaml
+  CA_CERT_NAME=${CA_NAME}-$(echo $DOMAIN | tr -s . -)-${CA_PORT}.pem
+
+  create_msp_config_yaml ${CA_NAME} ${CA_CERT_NAME} ${ORG_ADMIN_DIR}/msp
+
+  # private keys are hashed by name, but we only support one enrollment.
+  # test-network examples refer to this as "server.key", which is incorrect.
+  # This is the private key used to endorse transactions using the admin's
+  # public key.
+  mv ${ORG_ADMIN_DIR}/msp/keystore/*_sk ${ORG_ADMIN_DIR}/msp/keystore/key.pem
+}
+
+# create an enrollment MSP config.yaml
+function create_msp_config_yaml() {
+  local ca_name=$1
+  local ca_cert_name=$2
+  local msp_dir=$3
+  echo "Creating msp config ${msp_dir}/config.yaml with cert ${ca_cert_name}"
+
+  cat << EOF > ${msp_dir}/config.yaml
+NodeOUs:
+  Enable: true
+  ClientOUIdentifier:
+    Certificate: cacerts/${ca_cert_name}
+    OrganizationalUnitIdentifier: client
+  PeerOUIdentifier:
+    Certificate: cacerts/${ca_cert_name}
+    OrganizationalUnitIdentifier: peer
+  AdminOUIdentifier:
+    Certificate: cacerts/${ca_cert_name}
+    OrganizationalUnitIdentifier: admin
+  OrdererOUIdentifier:
+    Certificate: cacerts/${ca_cert_name}
+    OrganizationalUnitIdentifier: orderer
+EOF
 }

 function create_channel_MSP() {
@ -47,155 +148,124 @@ function create_channel_MSP() {
  create_channel_org_MSP org1 peer
  create_channel_org_MSP org2 peer

-  pop_fn
-}
-
-function aggregate_channel_MSP() {
-  push_fn "Aggregating channel MSP"
-
-  rm -rf ./build/msp/
-  mkdir -p ./build/msp
-  
-  kubectl -n $NS exec deploy/org0-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/ordererOrganizations/org0.example.com/msp > build/msp/msp-org0.example.com.tgz
-  kubectl -n $NS exec deploy/org1-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/peerOrganizations/org1.example.com/msp > build/msp/msp-org1.example.com.tgz
-  kubectl -n $NS exec deploy/org2-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/peerOrganizations/org2.example.com/msp > build/msp/msp-org2.example.com.tgz
-
-  kubectl -n $NS delete configmap msp-config || true
-  kubectl -n $NS create configmap msp-config --from-file=build/msp/
+  extract_orderer_tls_cert org0 orderer1
+  extract_orderer_tls_cert org0 orderer2
+  extract_orderer_tls_cert org0 orderer3

  pop_fn
 }

-function launch_admin_CLIs() {
-  push_fn "Launching admin CLIs"
+function create_channel_org_MSP() {
+  local org=$1
+  local type=$2
+  local ca_name=${org}-ca

-  launch kube/org0/org0-admin-cli.yaml
-  launch kube/org1/org1-admin-cli.yaml
-  launch kube/org2/org2-admin-cli.yaml
+  ORG_MSP_DIR=${TEMP_DIR}/channel-msp/${type}Organizations/${org}/msp
+  mkdir -p ${ORG_MSP_DIR}/cacerts
+  mkdir -p ${ORG_MSP_DIR}/tlscacerts

-  kubectl -n $NS rollout status deploy/org0-admin-cli
-  kubectl -n $NS rollout status deploy/org1-admin-cli
-  kubectl -n $NS rollout status deploy/org2-admin-cli
+  # extract the CA's signing authority from the CA/cainfo response
+  curl -s \
+    --cacert ${TEMP_DIR}/cas/${ca_name}/tls-cert.pem \
+    https://${ca_name}.${DOMAIN}/cainfo \
+    | jq -r .result.CAChain \
+    | base64 -d \
+    > ${ORG_MSP_DIR}/cacerts/ca-signcert.pem

-  pop_fn
+  # extract the CA's TLS CA certificate from the cert-manager secret
+  kubectl -n $NS get secret ${ca_name}-tls-cert -o json \
+    | jq -r .data.\"ca.crt\" \
+    | base64 -d \
+    > ${ORG_MSP_DIR}/tlscacerts/tlsca-signcert.pem
+
+  # create an MSP config.yaml with the CA's signing certificate
+  create_msp_config_yaml ${ca_name} ca-signcert.pem ${ORG_MSP_DIR}
+}
+
+# Extract an orderer's TLS signing certificate for inclusion in the channel config block
+function extract_orderer_tls_cert() {
+  local org=$1
+  local orderer=$2
+
+  echo "Extracting TLS cert for $org $orderer"
+
+  ORDERER_TLS_DIR=${TEMP_DIR}/channel-msp/ordererOrganizations/${org}/orderers/${org}-${orderer}/tls
+  mkdir -p $ORDERER_TLS_DIR/signcerts
+
+  kubectl -n $NS get secret ${org}-${orderer}-tls-cert -o json \
+    | jq -r .data.\"tls.crt\" \
+    | base64 -d \
+    > ${ORDERER_TLS_DIR}/signcerts/tls-cert.pem
 }

 function create_genesis_block() {
-  push_fn "Creating channel \"${CHANNEL_NAME}\""
+  push_fn "Creating channel genesis block"

-  echo 'set -x
-  configtxgen -profile TwoOrgsApplicationGenesis -channelID '${CHANNEL_NAME}' -outputBlock genesis_block.pb
-  # configtxgen -inspectBlock genesis_block.pb
+  FABRIC_CFG_PATH=${PWD}/config/org0 \
+    configtxgen \
+      -profile      TwoOrgsApplicationGenesis \
+      -channelID    $CHANNEL_NAME \
+      -outputBlock  ${TEMP_DIR}/genesis_block.pb

-  osnadmin channel join --orderer-address org0-orderer1:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb
-  osnadmin channel join --orderer-address org0-orderer2:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb
-  osnadmin channel join --orderer-address org0-orderer3:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb
+  # configtxgen -inspectBlock ${TEMP_DIR}/genesis_block.pb

-  ' | exec kubectl -n $NS exec deploy/org0-admin-cli -i -- /bin/bash
+  pop_fn
+}

-  # todo: readiness / liveiness equivalent for channel ?    Needs a little bit to settle before peers can join. 
+function join_channel_orderers() {
+  push_fn "Joining orderers to channel ${CHANNEL_NAME}"
+
+  join_channel_orderer org0 orderer1
+  join_channel_orderer org0 orderer2
+  join_channel_orderer org0 orderer3
+
+  # todo: readiness / liveiness equivalent for channel?  Needs a little bit to settle before peers can join.
  sleep 10

  pop_fn
 }

-function join_org_peers() {
+# Request from the channel ADMIN api that the orderer joins the target channel
+function join_channel_orderer() {
  local org=$1
-  push_fn "Joining ${org} peers to channel \"${CHANNEL_NAME}\""
+  local orderer=$2

-  echo 'set -x
-  # Fetch the genesis block from an orderer
-  peer channel \
-    fetch oldest \
-    genesis_block.pb \
-    -c '${CHANNEL_NAME}' \
-    -o org0-orderer1:6050 \
-    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
-
-  # Join peer1 to the channel.
-  CORE_PEER_ADDRESS='${org}'-peer1:7051 \
-  peer channel \
-    join \
-    -b genesis_block.pb \
-    -o org0-orderer1:6050 \
-    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
-
-  # Join peer2 to the channel.
-  CORE_PEER_ADDRESS='${org}'-peer2:7051 \
-  peer channel \
-    join \
-    -b genesis_block.pb \
-    -o org0-orderer1:6050 \
-    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
-
-  ' | exec kubectl -n $NS exec deploy/${org}-admin-cli -i -- /bin/bash
-
-  pop_fn
+  # The client certificate presented in this case is the admin user's enrollment key.  This is a stronger assertion
+  # of identity than the Docker Compose network, which transmits the orderer node's TLS key pair directly
+  osnadmin channel join \
+    --orderer-address ${org}-${orderer}-admin.${DOMAIN} \
+    --ca-file         ${TEMP_DIR}/channel-msp/ordererOrganizations/${org}/orderers/${org}-${orderer}/tls/signcerts/tls-cert.pem \
+    --client-cert     ${TEMP_DIR}/enrollments/${org}/users/${org}admin/msp/signcerts/cert.pem \
+    --client-key      ${TEMP_DIR}/enrollments/${org}/users/${org}admin/msp/keystore/key.pem \
+    --channelID       ${CHANNEL_NAME} \
+    --config-block    ${TEMP_DIR}/genesis_block.pb
 }

-function join_peers() {
+function join_channel_peers() {
  join_org_peers org1
  join_org_peers org2
 }

-# Copy the scripts/anchor_peers.sh to a remote volume
-function push_anchor_peer_script() {
+function join_org_peers() {
  local org=$1
+  push_fn "Joining ${org} peers to channel ${CHANNEL_NAME}"

-  tar cf - scripts/ | kubectl -n $NS exec -i -c main deploy/${org}-admin-cli -- tar xf - -C /var/hyperledger/fabric
-}
-
-verify_result() {
-  if [ $1 -ne 0 ]; then
-    echo $2
-    exit $1
-  fi
-}
-
-# Launch the anchor peer update script on a remote org admin CLI
-function invoke_anchor_peer_update() {
-  local org_num=$1
-  local peer_name=$2
-
-  kubectl exec \
-    -n $NS \
-    -c main \
-    deploy/org${org_num}-admin-cli \
-    -i \
-    /bin/bash -c "/var/hyperledger/fabric/scripts/set_anchor_peer.sh ${org_num} ${CHANNEL_NAME} ${peer_name}"
-
-  verify_result $? "Error updating anchor peer for org ${org_num}"
-}
-
-#
-# To update the anchor peers we will need to execute a script on each of the peer admin CLI containers.  These
-# commands can be individually piped into kubectl exec ... but it will be simpler if we transfer the anchor
-# peer update script over to the org volume and then trigger it from kubectl.
-#
-function update_anchor_peers() {
-  local peer_name=$1
-  push_fn "Updating anchor peers to ${peer_name}"
-
-  push_anchor_peer_script org1
-  push_anchor_peer_script org2
-
-  invoke_anchor_peer_update 1 ${peer_name}
-  invoke_anchor_peer_update 2 ${peer_name}
+  # Join peers to channel
+  join_channel_peer $org peer1
+  join_channel_peer $org peer2

  pop_fn
 }

-function channel_up() {
+function join_channel_peer() {
+  local org=$1
+  local peer=$2

-  create_channel_MSP
-  aggregate_channel_MSP
-  launch_admin_CLIs
+  export_peer_context $org $peer

-  create_genesis_block
-  join_peers
-
-  # peer1 was set as the anchor peer in configtx.yaml.  Setting this again will force an
-  # error to be returned from the channel up.  We might want to render the warning in
-  # this case to indicate that the call was made but had a nonzero exit. 
-  # update_anchor_peers peer1
+  peer channel join \
+    --blockpath ${TEMP_DIR}/genesis_block.pb \
+    --orderer   org0-orderer1.${DOMAIN} \
+    --tls        \
+    --cafile    ${TEMP_DIR}/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem
 }
--- a/test-network-k8s/scripts/fabric_CAs.sh
+++ b/test-network-k8s/scripts/fabric_CAs.sh
@ -5,27 +5,19 @@
 # SPDX-License-Identifier: Apache-2.0
 #

-function launch_CA() {
-  local yaml=$1
-  cat ${yaml} \
-    | sed 's,{{FABRIC_CONTAINER_REGISTRY}},'${FABRIC_CONTAINER_REGISTRY}',g' \
-    | sed 's,{{FABRIC_CA_VERSION}},'${FABRIC_CA_VERSION}',g' \
-    | kubectl -n $NS apply -f -
-}
-
 function launch_ECert_CAs() {
  push_fn "Launching Fabric CAs"

-  launch_CA kube/org0/org0-ca.yaml
-  launch_CA kube/org1/org1-ca.yaml
-  launch_CA kube/org2/org2-ca.yaml
+  apply_template kube/org0/org0-ca.yaml
+  apply_template kube/org1/org1-ca.yaml
+  apply_template kube/org2/org2-ca.yaml

  kubectl -n $NS rollout status deploy/org0-ca
  kubectl -n $NS rollout status deploy/org1-ca
  kubectl -n $NS rollout status deploy/org2-ca

  # todo: this papers over a nasty bug whereby the CAs are ready, but sporadically refuse connections after a down / up
-  # sleep 10
+  sleep 5

  pop_fn
 }
--- a/test-network-k8s/scripts/kind.sh
+++ b/test-network-k8s/scripts/kind.sh
@ -31,7 +31,7 @@ function load_docker_images() {
 }

 function apply_nginx_ingress() {
-  push_fn "Launching Nginx ingress controller"
+  push_fn "Launching ingress controller"

  # This ingress-nginx.yaml was generated 9/24 from https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
  # with modifications for ssl-passthrough required to launch IBP-support with the nginx ingress.
@ -42,12 +42,29 @@ function apply_nginx_ingress() {
  pop_fn
 }

-function install_cert_manager() {
-  push_fn "Installing cert-manager"
+function wait_for_nginx_ingress() {
+  push_fn "Waiting for ingress controller"

-    # Install cert-manager to manage TLS certificates
+  kubectl wait --namespace ingress-nginx \
+    --for=condition=ready pod \
+    --selector=app.kubernetes.io/component=controller \
+    --timeout=90s
+
+  pop_fn
+}
+
+function apply_cert_manager() {
+  push_fn "Launching cert-manager"
+
+  # Install cert-manager to manage TLS certificates
  kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.6.1/cert-manager.yaml

+  pop_fn
+}
+
+function wait_for_cert_manager() {
+  push_fn "Waiting for cert-manager"
+
  kubectl -n cert-manager rollout status deploy/cert-manager
  kubectl -n cert-manager rollout status deploy/cert-manager-cainjector
  kubectl -n cert-manager rollout status deploy/cert-manager-webhook
@ -58,6 +75,9 @@ function install_cert_manager() {
 function kind_create() {
  push_fn  "Creating cluster \"${CLUSTER_NAME}\""

+  # prevent the next kind cluster from using the previous Fabric network's enrollments.
+  rm -rf $PWD/build
+
  # todo: always delete?  Maybe return no-op if the cluster already exists?
  kind delete cluster --name $CLUSTER_NAME

@ -87,8 +107,8 @@ nodes:
      - containerPort: 443
        hostPort: ${ingress_https_port}
        protocol: TCP
-networking:
-  kubeProxyMode: "ipvs"
+#networking:
+#  kubeProxyMode: "ipvs"

 # create a cluster with the local registry enabled in containerd
 containerdConfigPatches:
@ -98,6 +118,12 @@ containerdConfigPatches:

 EOF

+  # workaround for https://github.com/hyperledger/fabric-samples/issues/550 - pods can not resolve external DNS
+  for node in $(kind get nodes);
+  do
+    docker exec "$node" sysctl net.ipv4.conf.all.route_localnet=1;
+  done
+
  pop_fn
 }

@ -151,13 +177,16 @@ function kind_init() {

  kind_create
  apply_nginx_ingress
-  install_cert_manager
+  apply_cert_manager
  launch_docker_registry

  if [ "${STAGE_DOCKER_IMAGES}" == true ]; then
    pull_docker_images 
    load_docker_images 
  fi
+
+  wait_for_cert_manager
+  wait_for_nginx_ingress
 }

 function kind_unkind() {
--- a/test-network-k8s/scripts/prereqs.sh
+++ b/test-network-k8s/scripts/prereqs.sh
@ -8,6 +8,8 @@
 # Double check that kind, kubectl, docker, and all required images are present.
 function check_prereqs() {

+  set +e
+
  ${CONTAINER_CLI} version > /dev/null
  if [[ $? -ne 0 ]]; then
    echo "No '${CONTAINER_CLI}' binary available?"
@ -31,4 +33,27 @@ function check_prereqs() {
    echo "No 'jq' binary available? (https://stedolan.github.io/jq/)"
    exit 1
  fi
+
+  # Use the local fabric binaries if available.  If not, go get them.
+  bin/peer version &> /dev/null
+  if [[ $? -ne 0 ]]; then
+    echo "Downloading LATEST Fabric binaries and config"
+    curl -sSL https://raw.githubusercontent.com/hyperledger/fabric/main/scripts/bootstrap.sh | bash -s -- -s -d
+
+    # remove sample config files extracted by the installation script
+    rm config/configtx.yaml
+    rm config/core.yaml
+    rm config/orderer.yaml
+  fi
+
+  export PATH=bin:$PATH
+
+  # Double-check that the binary transfer was OK
+  peer version > /dev/null
+  if [[ $? -ne 0 ]]; then
+    log "No 'peer' binary available?"
+    exit 1
+  fi
+
+  set -e
 }
--- a/test-network-k8s/scripts/rest_sample.sh
+++ b/test-network-k8s/scripts/rest_sample.sh
@ -5,16 +5,8 @@
 # SPDX-License-Identifier: Apache-2.0
 #

-function extract_MSP_archives() {
-  mkdir -p build/msp
-
-  kubectl -n $NS exec deploy/org1-ca -- tar zcf - -C /var/hyperledger/fabric organizations/peerOrganizations/org1.example.com/msp | tar zxf - -C build/msp
-  kubectl -n $NS exec deploy/org2-ca -- tar zcf - -C /var/hyperledger/fabric organizations/peerOrganizations/org2.example.com/msp | tar zxf - -C build/msp
-
-  kubectl -n $NS exec deploy/org1-ca -- tar zcf - -C /var/hyperledger/fabric organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp | tar zxf - -C build/msp
-  kubectl -n $NS exec deploy/org2-ca -- tar zcf - -C /var/hyperledger/fabric organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp | tar zxf - -C build/msp
-}
-
+# This magical awk script led to 30 hours of debugging a "TLS handshake error"
+# moral: do not edit / alter the number of '\' in the following transform:
 function one_line_pem {
    echo "`awk 'NF {sub(/\\n/, ""); printf "%s\\\\\\\n",$0;}' $1`"
 }
@ -32,41 +24,32 @@ function json_ccp {
 function construct_rest_sample_configmap() {
  push_fn "Constructing fabric-rest-sample connection profiles"

-  extract_MSP_archives
+  ENROLLMENT_DIR=${TEMP_DIR}/enrollments
+  CHANNEL_MSP_DIR=${TEMP_DIR}/channel-msp
+  CONFIG_DIR=${TEMP_DIR}/fabric-rest-sample-config 

-  mkdir -p build/fabric-rest-sample-config
-
-  local peer_pem=build/msp/organizations/peerOrganizations/org1.example.com/msp/tlscacerts/org1-tls-ca.pem
-  local ca_pem=build/msp/organizations/peerOrganizations/org1.example.com/msp/cacerts/org1-ca.pem
+  mkdir -p $CONFIG_DIR

+  local peer_pem=$CHANNEL_MSP_DIR/peerOrganizations/org1/msp/tlscacerts/tlsca-signcert.pem
+  local ca_pem=$CHANNEL_MSP_DIR/peerOrganizations/org1/msp/cacerts/ca-signcert.pem
  echo "$(json_ccp 1 $peer_pem $ca_pem)" > build/fabric-rest-sample-config/HLF_CONNECTION_PROFILE_ORG1

-  peer_pem=build/msp/organizations/peerOrganizations/org2.example.com/msp/tlscacerts/org2-tls-ca.pem
-  ca_pem=build/msp/organizations/peerOrganizations/org2.example.com/msp/cacerts/org2-ca.pem
-
+  peer_pem=$CHANNEL_MSP_DIR/peerOrganizations/org2/msp/tlscacerts/tlsca-signcert.pem
+  ca_pem=$CHANNEL_MSP_DIR/peerOrganizations/org2/msp/cacerts/ca-signcert.pem
  echo "$(json_ccp 2 $peer_pem $ca_pem)" > build/fabric-rest-sample-config/HLF_CONNECTION_PROFILE_ORG2

-  cat build/msp/organizations/peerOrganizations/org1.example.com/users/Admin\@org1.example.com/msp/signcerts/cert.pem > build/fabric-rest-sample-config/HLF_CERTIFICATE_ORG1
-  cat build/msp/organizations/peerOrganizations/org2.example.com/users/Admin\@org2.example.com/msp/signcerts/cert.pem > build/fabric-rest-sample-config/HLF_CERTIFICATE_ORG2
+  cp $ENROLLMENT_DIR/org1/users/org1admin/msp/signcerts/cert.pem $CONFIG_DIR/HLF_CERTIFICATE_ORG1
+  cp $ENROLLMENT_DIR/org2/users/org2admin/msp/signcerts/cert.pem $CONFIG_DIR/HLF_CERTIFICATE_ORG2

-  cat build/msp/organizations/peerOrganizations/org1.example.com/users/Admin\@org1.example.com/msp/keystore/server.key > build/fabric-rest-sample-config/HLF_PRIVATE_KEY_ORG1
-  cat build/msp/organizations/peerOrganizations/org2.example.com/users/Admin\@org2.example.com/msp/keystore/server.key > build/fabric-rest-sample-config/HLF_PRIVATE_KEY_ORG2
+  cp $ENROLLMENT_DIR/org1/users/org1admin/msp/keystore/key.pem $CONFIG_DIR/HLF_PRIVATE_KEY_ORG1
+  cp $ENROLLMENT_DIR/org2/users/org2admin/msp/keystore/key.pem $CONFIG_DIR/HLF_PRIVATE_KEY_ORG2

  kubectl -n $NS delete configmap fabric-rest-sample-config || true
-  kubectl -n $NS create configmap fabric-rest-sample-config --from-file=build/fabric-rest-sample-config/
+  kubectl -n $NS create configmap fabric-rest-sample-config --from-file=$CONFIG_DIR

  pop_fn
 }

-# todo: Make sure to port this to IKS / ICP
-function ensure_rest_sample_image() {
-  push_fn "Ensuring fabric-rest-sample image"
-
-  # todo: apply a tag / label to avoid pulling :latest from ghcr.io
-
-  pop_fn 0
-}
-
 function rollout_rest_sample() {
  push_fn "Starting fabric-rest-sample"

@ -77,15 +60,19 @@ function rollout_rest_sample() {
 }

 function launch_rest_sample() {
-  ensure_rest_sample_image
+
  construct_rest_sample_configmap
-  rollout_rest_sample
+
+  apply_template kube/fabric-rest-sample.yaml
+
+  kubectl -n $NS rollout status deploy/fabric-rest-sample

  log ""
-  log "The fabric-rest-sample has started.  See https://github.com/hyperledger/fabric-samples/tree/main/asset-transfer-basic/rest-api-typescript for additional usage."
+  log "The fabric-rest-sample has started."
+  log "See https://github.com/hyperledger/fabric-samples/tree/main/asset-transfer-basic/rest-api-typescript for additional usage details."
  log "To access the endpoint:"
  log ""
  log "export SAMPLE_APIKEY=97834158-3224-4CE7-95F9-A148C886653E"
-  log 'curl -s --header "X-Api-Key: ${SAMPLE_APIKEY}" http://localhost/api/assets'
+  log 'curl -s --header "X-Api-Key: ${SAMPLE_APIKEY}" http://fabric-rest-sample.'${DOMAIN}'/api/assets'
  log ""
 }
--- a/test-network-k8s/scripts/test_network.sh
+++ b/test-network-k8s/scripts/test_network.sh
@ -5,20 +5,12 @@
 # SPDX-License-Identifier: Apache-2.0
 #

-function launch() {
-  local yaml=$1
-  cat ${yaml} \
-    | sed 's,{{FABRIC_CONTAINER_REGISTRY}},'${FABRIC_CONTAINER_REGISTRY}',g' \
-    | sed 's,{{FABRIC_VERSION}},'${FABRIC_VERSION}',g' \
-    | kubectl -n $NS apply -f -
-}
-
 function launch_orderers() {
  push_fn "Launching orderers"

-  launch kube/org0/org0-orderer1.yaml
-  launch kube/org0/org0-orderer2.yaml
-  launch kube/org0/org0-orderer3.yaml
+  apply_template kube/org0/org0-orderer1.yaml
+  apply_template kube/org0/org0-orderer2.yaml
+  apply_template kube/org0/org0-orderer3.yaml

  kubectl -n $NS rollout status deploy/org0-orderer1
  kubectl -n $NS rollout status deploy/org0-orderer2
@ -30,10 +22,10 @@ function launch_orderers() {
 function launch_peers() {
  push_fn "Launching peers"

-  launch kube/org1/org1-peer1.yaml
-  launch kube/org1/org1-peer2.yaml
-  launch kube/org2/org2-peer1.yaml
-  launch kube/org2/org2-peer2.yaml
+  apply_template kube/org1/org1-peer1.yaml
+  apply_template kube/org1/org1-peer2.yaml
+  apply_template kube/org2/org2-peer1.yaml
+  apply_template kube/org2/org2-peer2.yaml

  kubectl -n $NS rollout status deploy/org1-peer1
  kubectl -n $NS rollout status deploy/org1-peer2
@ -43,155 +35,127 @@ function launch_peers() {
  pop_fn
 }

-function create_org0_local_MSP() {
-  echo 'set -x
+# todo: enroll org admin LOCALLY from the host OS
+#  fabric-ca-client register --id.name org1-admin --id.secret org1adminpw  --id.type admin   --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
+#  fabric-ca-client enroll --url https://org1-admin:org1adminpw@org1-ca  --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/server.key
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/config.yaml
+
+#  fabric-ca-client register --id.name org1-admin --id.secret org1adminpw  --id.type admin   --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
+#  fabric-ca-client enroll --url https://org1-admin:org1adminpw@org1-ca  --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/server.key
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/config.yaml
+
+#  fabric-ca-client register --id.name org2-admin --id.secret org2adminpw  --id.type admin   --url https://org2-ca --mspdir $FABRIC_CA_CLIENT_HOME/org2-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
+#  fabric-ca-client enroll --url https://org2-admin:org2adminpw@org2-ca  --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/keystore/server.key
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/config.yaml
+
+
+# Each network node needs a registration, enrollment, and MSP config.yaml
+function create_node_local_MSP() {
+  local node_type=$1
+  local org=$2
+  local node=$3
+  local csr_hosts=$4
+  local id_name=${org}-${node}
+  local id_secret=${node_type}pw
+  local ca_name=${org}-ca
+
+  cat <<EOF | kubectl -n $NS exec deploy/${ca_name} -i -- /bin/sh
+
+  set -x
  export FABRIC_CA_CLIENT_HOME=/var/hyperledger/fabric-ca-client
  export FABRIC_CA_CLIENT_TLS_CERTFILES=/var/hyperledger/fabric/config/tls/ca.crt

  # Each identity in the network needs a registration and enrollment.
-  fabric-ca-client register --id.name org0-orderer1 --id.secret ordererpw --id.type orderer --url https://org0-ca --mspdir $FABRIC_CA_CLIENT_HOME/org0-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org0-orderer2 --id.secret ordererpw --id.type orderer --url https://org0-ca --mspdir $FABRIC_CA_CLIENT_HOME/org0-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org0-orderer3 --id.secret ordererpw --id.type orderer --url https://org0-ca --mspdir $FABRIC_CA_CLIENT_HOME/org0-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org0-admin --id.secret org0adminpw  --id.type admin   --url https://org0-ca --mspdir $FABRIC_CA_CLIENT_HOME/org0-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
+  fabric-ca-client register \
+    --id.name ${id_name} \
+    --id.secret ${id_secret} \
+    --id.type ${node_type} \
+    --url https://${ca_name} \
+    --mspdir /var/hyperledger/fabric-ca-client/${ca_name}/rcaadmin/msp

-  fabric-ca-client enroll --url https://org0-orderer1:ordererpw@org0-ca --csr.hosts org0-orderer1 --mspdir /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp
-  fabric-ca-client enroll --url https://org0-orderer2:ordererpw@org0-ca --csr.hosts org0-orderer2 --mspdir /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/msp
-  fabric-ca-client enroll --url https://org0-orderer3:ordererpw@org0-ca --csr.hosts org0-orderer3 --mspdir /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/msp
-  fabric-ca-client enroll --url https://org0-admin:org0adminpw@org0-ca --mspdir /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/users/Admin@org0.example.com/msp
-
-  # Create an MSP config.yaml (why is this not generated by the enrollment by fabric-ca-client?)
-  echo "NodeOUs:
-    Enable: true
-    ClientOUIdentifier:
-      Certificate: cacerts/org0-ca.pem
-      OrganizationalUnitIdentifier: client
-    PeerOUIdentifier:
-      Certificate: cacerts/org0-ca.pem
-      OrganizationalUnitIdentifier: peer
-    AdminOUIdentifier:
-      Certificate: cacerts/org0-ca.pem
-      OrganizationalUnitIdentifier: admin
-    OrdererOUIdentifier:
-      Certificate: cacerts/org0-ca.pem
-      OrganizationalUnitIdentifier: orderer" > /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp/config.yaml
-
-  cp /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/msp/config.yaml
-  cp /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/msp/config.yaml
-  ' | exec kubectl -n $NS exec deploy/org0-ca -i -- /bin/sh
-}
-
-function create_org1_local_MSP() {
-
-  echo 'set -x
-  export FABRIC_CA_CLIENT_HOME=/var/hyperledger/fabric-ca-client
-  export FABRIC_CA_CLIENT_TLS_CERTFILES=/var/hyperledger/fabric/config/tls/ca.crt
-
-  # Each identity in the network needs a registration and enrollment.
-  fabric-ca-client register --id.name org1-peer1 --id.secret peerpw --id.type peer --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org1-peer2 --id.secret peerpw --id.type peer --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org1-admin --id.secret org1adminpw  --id.type admin   --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
-
-  fabric-ca-client enroll --url https://org1-peer1:peerpw@org1-ca --csr.hosts localhost,org1-peer1,org1-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp
-  fabric-ca-client enroll --url https://org1-peer2:peerpw@org1-ca --csr.hosts localhost,org1-peer2,org1-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer2.org1.example.com/msp
-  fabric-ca-client enroll --url https://org1-admin:org1adminpw@org1-ca  --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp
-
-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/server.key
+  fabric-ca-client enroll \
+    --url https://${id_name}:${id_secret}@${ca_name} \
+    --csr.hosts ${csr_hosts} \
+    --mspdir /var/hyperledger/fabric/organizations/${node_type}Organizations/${org}.example.com/${node_type}s/${id_name}.${org}.example.com/msp

  # Create local MSP config.yaml
  echo "NodeOUs:
    Enable: true
    ClientOUIdentifier:
-      Certificate: cacerts/org1-ca.pem
+      Certificate: cacerts/${org}-ca.pem
      OrganizationalUnitIdentifier: client
    PeerOUIdentifier:
-      Certificate: cacerts/org1-ca.pem
+      Certificate: cacerts/${org}-ca.pem
      OrganizationalUnitIdentifier: peer
    AdminOUIdentifier:
-      Certificate: cacerts/org1-ca.pem
+      Certificate: cacerts/${org}-ca.pem
      OrganizationalUnitIdentifier: admin
    OrdererOUIdentifier:
-      Certificate: cacerts/org1-ca.pem
-      OrganizationalUnitIdentifier: orderer" > /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml
-
-
-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer2.org1.example.com/msp/config.yaml
-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/config.yaml
-  ' | exec kubectl -n $NS exec deploy/org1-ca -i -- /bin/sh
-
+      Certificate: cacerts/${org}-ca.pem
+      OrganizationalUnitIdentifier: orderer" > /var/hyperledger/fabric/organizations/${node_type}Organizations/${org}.example.com/${node_type}s/${id_name}.${org}.example.com/msp/config.yaml
+EOF
 }

-function create_org2_local_MSP() {
-  echo 'set -x
-  export FABRIC_CA_CLIENT_HOME=/var/hyperledger/fabric-ca-client
-  export FABRIC_CA_CLIENT_TLS_CERTFILES=/var/hyperledger/fabric/config/tls/ca.crt
+function create_orderer_local_MSP() {
+  local org=$1
+  local orderer=$2
+  local csr_hosts=${org}-${orderer}

-  # Each identity in the network needs a registration and enrollment.
-  fabric-ca-client register --id.name org2-peer1 --id.secret peerpw --id.type peer --url https://org2-ca --mspdir $FABRIC_CA_CLIENT_HOME/org2-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org2-peer2 --id.secret peerpw --id.type peer --url https://org2-ca --mspdir $FABRIC_CA_CLIENT_HOME/org2-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org2-admin --id.secret org2adminpw  --id.type admin   --url https://org2-ca --mspdir $FABRIC_CA_CLIENT_HOME/org2-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
+  create_node_local_MSP orderer $org $orderer $csr_hosts
+}

-  fabric-ca-client enroll --url https://org2-peer1:peerpw@org2-ca --csr.hosts localhost,org2-peer1,org2-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp
-  fabric-ca-client enroll --url https://org2-peer2:peerpw@org2-ca --csr.hosts localhost,org2-peer2,org2-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer2.org2.example.com/msp
-  fabric-ca-client enroll --url https://org2-admin:org2adminpw@org2-ca  --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp
+function create_peer_local_MSP() {
+  local org=$1
+  local peer=$2
+  local csr_hosts=localhost,${org}-${peer},${org}-peer-gateway-svc

-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/keystore/server.key
-
-  # Create local MSP config.yaml
-  echo "NodeOUs:
-    Enable: true
-    ClientOUIdentifier:
-      Certificate: cacerts/org2-ca.pem
-      OrganizationalUnitIdentifier: client
-    PeerOUIdentifier:
-      Certificate: cacerts/org2-ca.pem
-      OrganizationalUnitIdentifier: peer
-    AdminOUIdentifier:
-      Certificate: cacerts/org2-ca.pem
-      OrganizationalUnitIdentifier: admin
-    OrdererOUIdentifier:
-      Certificate: cacerts/org2-ca.pem
-      OrganizationalUnitIdentifier: orderer" > /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp/config.yaml
-
-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer2.org2.example.com/msp/config.yaml
-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/config.yaml
-  ' | exec kubectl -n $NS exec deploy/org2-ca -i -- /bin/sh
+  create_node_local_MSP peer $org $peer $csr_hosts
 }

 function create_local_MSP() {
  push_fn "Creating local node MSP"

-  create_org0_local_MSP
-  create_org1_local_MSP
-  create_org2_local_MSP
-
-  pop_fn
-}
-
-# TLS certificates are isused by the CA's Issuer, stored in a Kube secret, and mounted into the pod at /var/hyperledger/fabric/config/tls.
-# For consistency with the Fabric-CA guide, his function copies the orderer's TLS certs into the traditional Fabric MSP / folder structure.
-function extract_orderer_tls_cert() {
-  local orderer=$1
-
-  echo 'set -x
-
-  mkdir -p /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/'${orderer}'.org0.example.com/tls/signcerts/
-
-  cp \
-    var/hyperledger/fabric/config/tls/tls.crt \
-    /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/'${orderer}'.org0.example.com/tls/signcerts/cert.pem
-
-  ' | exec kubectl -n $NS exec deploy/${orderer} -i -c main -- /bin/sh
-}
-
-function extract_orderer_tls_certs() {
-  push_fn "Extracting orderer TLS certs to local MSP folder"
-
-  extract_orderer_tls_cert org0-orderer1
-  extract_orderer_tls_cert org0-orderer2
-  extract_orderer_tls_cert org0-orderer3
+  create_orderer_local_MSP org0 orderer1
+  create_orderer_local_MSP org0 orderer2
+  create_orderer_local_MSP org0 orderer3
+
+  create_peer_local_MSP org1 peer1
+  create_peer_local_MSP org1 peer2
+
+  create_peer_local_MSP org2 peer1
+  create_peer_local_MSP org2 peer2

  pop_fn
 }
+#
+## TLS certificates are isused by the CA's Issuer, stored in a Kube secret, and mounted into the pod at /var/hyperledger/fabric/config/tls.
+## For consistency with the Fabric-CA guide, his function copies the orderer's TLS certs into the traditional Fabric MSP / folder structure.
+#function extract_orderer_tls_cert() {
+#  local orderer=$1
+#
+#  echo 'set -x
+#
+#  mkdir -p /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/'${orderer}'.org0.example.com/tls/signcerts/
+#
+#  cp \
+#    var/hyperledger/fabric/config/tls/tls.crt \
+#    /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/'${orderer}'.org0.example.com/tls/signcerts/cert.pem
+#
+#  ' | exec kubectl -n $NS exec deploy/${orderer} -i -c main -- /bin/sh
+#}
+#
+#function extract_orderer_tls_certs() {
+#  push_fn "Extracting orderer TLS certs to local MSP folder"
+#
+#  extract_orderer_tls_cert org0-orderer1
+#  extract_orderer_tls_cert org0-orderer2
+#  extract_orderer_tls_cert org0-orderer3
+#
+#  pop_fn
+#}

 function network_up() {

@ -213,17 +177,13 @@ function network_up() {
  launch_orderers
  launch_peers

-  extract_orderer_tls_certs
+#  extract_orderer_tls_certs
 }

 function stop_services() {
  push_fn "Stopping Fabric services"

-  # These pods are busy executing `sleep MAX_INT` and do not shut down very quickly...
-#  kubectl -n $NS delete deployment/org0-admin-cli --grace-period=0 --force
-#  kubectl -n $NS delete deployment/org1-admin-cli --grace-period=0 --force
-#  kubectl -n $NS delete deployment/org2-admin-cli --grace-period=0 --force
-
+  kubectl -n $NS delete ingress --all
  kubectl -n $NS delete deployment --all
  kubectl -n $NS delete pod --all
  kubectl -n $NS delete service --all
@ -252,4 +212,6 @@ function scrub_org_volumes() {
 function network_down() {
  stop_services
  scrub_org_volumes
+
+  rm -rf $PWD/build
 }
--- a/test-network-k8s/scripts/utils.sh
+++ b/test-network-k8s/scripts/utils.sh
@ -25,6 +25,8 @@ function logging_init() {
 function exit_fn() {
  rc=$?

+  set +x
+
  # Write an error icon to the current logging statement.
  if [ "0" -ne $rc ]; then
    pop_fn $rc
@ -73,3 +75,29 @@ function pop_fn() {
  echo "" >> ${LOG_FILE}
 }

+# Apply the current environment to a k8s template and apply to the cluster.
+function apply_template() {
+
+  echo "Applying template $1:"
+  cat $1 | envsubst
+
+  cat $1 | envsubst | kubectl -n $NS apply -f -
+}
+
+# Set the calling context to refer the peer binary to the correct org / peer instance
+#
+# todo: Expose the output of this function to a target that prints the context to STDOUT.
+#
+# e.g.:
+# bash $ source $(network set-peer-context org1 peer2)
+# bash $ peer chaincode list
+# bash $ ...
+function export_peer_context() {
+  local org=$1
+  local peer=$2
+
+  export FABRIC_CFG_PATH=${PWD}/config/${org}
+  export CORE_PEER_ADDRESS=${org}-${peer}.${DOMAIN}:443
+  export CORE_PEER_MSPCONFIGPATH=${TEMP_DIR}/enrollments/${org}/users/${org}admin/msp
+  export CORE_PEER_TLS_ROOTCERT_FILE=${TEMP_DIR}/channel-msp/peerOrganizations/${org}/msp/tlscacerts/tlsca-signcert.pem
+}