Access the test network services via a local Nginx ingress controller.

Signed-off-by: Josh Kneubuhl <jkneubuh@us.ibm.com>
2026-06-23 01:55:10 +00:00 · 2022-03-25 13:31:20 -04:00 · 2022-03-25 13:31:20 -04:00 · aeed596ec5
commit aeed596ec5
parent e1fb6269dc
23 changed files with 983 additions and 549 deletions
--- a/test-network-k8s/.gitignore
+++ b/test-network-k8s/.gitignore
@ -3,3 +3,4 @@ network.log
 network-debug.log 
 build/
 .env
+bin/
--- a/test-network-k8s/config/org0/configtx.yaml
+++ b/test-network-k8s/config/org0/configtx.yaml
@ -25,7 +25,7 @@ Organizations:
    ID: OrdererMSP

    # MSPDir is the filesystem path which contains the MSP configuration
-    MSPDir: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp
+    MSPDir: ../../build/channel-msp/ordererOrganizations/org0/msp

    # Policies defines the set of policies at this level of the config tree
    # For organization policies, their canonical path is usually
@ -54,7 +54,7 @@ Organizations:
    # ID to load the MSP definition as
    ID: Org1MSP

-    MSPDir: /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/msp
+    MSPDir: ../../build/channel-msp/peerOrganizations/org1/msp

    # Policies defines the set of policies at this level of the config tree
    # For organization policies, their canonical path is usually
@ -89,7 +89,7 @@ Organizations:
    # ID to load the MSP definition as
    ID: Org2MSP

-    MSPDir: /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/msp
+    MSPDir: ../../build/channel-msp/peerOrganizations/org2/msp

    # Policies defines the set of policies at this level of the config tree
    # For organization policies, their canonical path is usually
@ -224,16 +224,16 @@ Orderer: &OrdererDefaults
    Consenters:
      - Host: org0-orderer1
        Port: 6050
-        ClientTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/tls/signcerts/cert.pem
-        ServerTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/tls/signcerts/cert.pem
+        ClientTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem
+        ServerTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem
      - Host: org0-orderer2
        Port: 6050
-        ClientTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/tls/signcerts/cert.pem
-        ServerTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/tls/signcerts/cert.pem
+        ClientTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer2/tls/signcerts/tls-cert.pem
+        ServerTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer2/tls/signcerts/tls-cert.pem
      - Host: org0-orderer3
        Port: 6050
-        ClientTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/tls/signcerts/cert.pem
-        ServerTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/tls/signcerts/cert.pem
+        ClientTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer3/tls/signcerts/tls-cert.pem
+        ServerTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer3/tls/signcerts/tls-cert.pem


    # Options to be specified for all the etcd/raft nodes. The values here
--- a/test-network-k8s/kube/org0/org0-admin-cli.yaml
+++ b/test-network-k8s/kube/org0/org0-admin-cli.yaml
@ -1,61 +0,0 @@
-#
-# Copyright IBM Corp. All Rights Reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: org0-admin-cli
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: org0-admin-cli
-  template:
-    metadata:
-      labels:
-        app: org0-admin-cli
-    spec:
-      containers:
-        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-tools:{{FABRIC_VERSION}}
-          imagePullPolicy: IfNotPresent
-          env:
-            - name: FABRIC_CFG_PATH
-              value: /var/hyperledger/fabric/config
-          args:
-            - sleep
-            - "2147483647"
-          workingDir: /root
-          volumeMounts:
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-            - name: fabric-config
-              mountPath: /var/hyperledger/fabric/config
-
-      # This init container will unfurl all of the MSP archives listed in the msp-config config map.
-      initContainers:
-        - name: msp-unfurl
-          image: busybox
-          command:
-            - sh
-            - -c
-            - "for msp in $(ls /msp/msp-*.tgz); do echo $msp && tar zxvf $msp -C /var/hyperledger/fabric; done"
-          volumeMounts:
-            - name: msp-config
-              mountPath: /msp
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-
-      volumes:
-        - name: fabric-volume
-          persistentVolumeClaim:
-            claimName: fabric-org0
-        - name: fabric-config
-          configMap:
-            name: org0-config
-        - name: msp-config
-          configMap:
-            name: msp-config
--- a/test-network-k8s/kube/org0/org0-ca.yaml
+++ b/test-network-k8s/kube/org0/org0-ca.yaml
@ -17,6 +17,7 @@ spec:
    - localhost
    - org0-ca
    - org0-ca.test-network.svc.cluster.local
+    - org0-ca.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org0-ca-tls-cert
@ -40,7 +41,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-ca:{{FABRIC_CA_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-ca:${FABRIC_CA_VERSION}
          imagePullPolicy: IfNotPresent
          env:
            - name: FABRIC_CA_SERVER_CA_NAME
@ -89,8 +90,35 @@ metadata:
  name: org0-ca
 spec:
  ports:
-    - name: tls
+    - name: https
      port: 443
      protocol: TCP
  selector:
-    app: org0-ca
+    app: org0-ca
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org0-ca
+  name: org0-ca
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org0-ca.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-ca
+                port:
+                  name: https
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+      - org0-ca.${DOMAIN}
--- a/test-network-k8s/kube/org0/org0-orderer1.yaml
+++ b/test-network-k8s/kube/org0/org0-orderer1.yaml
@ -9,7 +9,7 @@ apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
  name: org0-orderer1-tls-cert
-  namespace: test-network
+  namespace: ${NS}
 spec:
  isCA: false
  privateKey:
@ -18,7 +18,9 @@ spec:
  dnsNames:
    - localhost
    - org0-orderer1
-    - org0-orderer1.test-network.svc.cluster.local
+    - org0-orderer1.${NS}.svc.cluster.local
+    - org0-orderer1.${DOMAIN}
+    - org0-orderer1-admin.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org0-orderer1-tls-cert
@ -42,6 +44,12 @@ data:
  ORDERER_GENERAL_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
  ORDERER_GENERAL_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
  ORDERER_GENERAL_BOOTSTRAPMETHOD: none
+  ORDERER_ADMIN_TLS_ENABLED: "true"
+  ORDERER_ADMIN_TLS_CERTIFICATE: /var/hyperledger/fabric/config/tls/tls.crt
+  ORDERER_ADMIN_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
+  ORDERER_ADMIN_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
+  # Authenticate client connections with the org's ecert / admin user enrollments
+  ORDERER_ADMIN_TLS_CLIENTROOTCAS: "[/var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp/cacerts/org0-ca.pem]"
  ORDERER_FILELEDGER_LOCATION: /var/hyperledger/fabric/data/orderer1
  ORDERER_CONSENSUS_WALDIR: /var/hyperledger/fabric/data/orderer1/etcdraft/wal
  ORDERER_CONSENSUS_SNAPDIR: /var/hyperledger/fabric/data/orderer1/etcdraft/wal
@ -65,7 +73,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-orderer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-orderer:${FABRIC_VERSION}
          imagePullPolicy: IfNotPresent
          envFrom:
            - configMapRef:
@ -109,4 +117,43 @@ spec:
      port: 9443
      protocol: TCP
  selector:
-    app: org0-orderer1
+    app: org0-orderer1
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org0-orderer1
+  name: org0-orderer1
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org0-orderer1.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer1
+                port:
+                  name: general
+            path: /
+            pathType: ImplementationSpecific
+    - host: org0-orderer1-admin.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer1
+                port:
+                  name: admin
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org0-orderer1.${DOMAIN}
+    - hosts:
+        - org0-orderer1-admin.${DOMAIN}
--- a/test-network-k8s/kube/org0/org0-orderer2.yaml
+++ b/test-network-k8s/kube/org0/org0-orderer2.yaml
@ -3,12 +3,13 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 #
+
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
  name: org0-orderer2-tls-cert
-  namespace: test-network
+  namespace: ${NS}
 spec:
  isCA: false
  privateKey:
@ -17,7 +18,9 @@ spec:
  dnsNames:
    - localhost
    - org0-orderer2
-    - org0-orderer2.test-network.svc.cluster.local
+    - org0-orderer2.${NS}.svc.cluster.local
+    - org0-orderer2.${DOMAIN}
+    - org0-orderer2-admin.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org0-orderer2-tls-cert
@ -41,6 +44,12 @@ data:
  ORDERER_GENERAL_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
  ORDERER_GENERAL_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
  ORDERER_GENERAL_BOOTSTRAPMETHOD: none
+  ORDERER_ADMIN_TLS_ENABLED: "true"
+  ORDERER_ADMIN_TLS_CERTIFICATE: /var/hyperledger/fabric/config/tls/tls.crt
+  ORDERER_ADMIN_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
+  ORDERER_ADMIN_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
+  # Authenticate client connections with the org's ecert / admin user enrollments
+  ORDERER_ADMIN_TLS_CLIENTROOTCAS: "[/var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/msp/cacerts/org0-ca.pem]"
  ORDERER_FILELEDGER_LOCATION: /var/hyperledger/fabric/data/orderer2
  ORDERER_CONSENSUS_WALDIR: /var/hyperledger/fabric/data/orderer2/etcdraft/wal
  ORDERER_CONSENSUS_SNAPDIR: /var/hyperledger/fabric/data/orderer2/etcdraft/wal
@ -64,7 +73,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-orderer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-orderer:${FABRIC_VERSION}
          imagePullPolicy: IfNotPresent
          envFrom:
            - configMapRef:
@ -108,4 +117,43 @@ spec:
      port: 9443
      protocol: TCP
  selector:
-    app: org0-orderer2
+    app: org0-orderer2
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org0-orderer2
+  name: org0-orderer2
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org0-orderer2.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer2
+                port:
+                  name: general
+            path: /
+            pathType: ImplementationSpecific
+    - host: org0-orderer2-admin.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer2
+                port:
+                  name: admin
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org0-orderer2.${DOMAIN}
+    - hosts:
+        - org0-orderer2-admin.${DOMAIN}
--- a/test-network-k8s/kube/org0/org0-orderer3.yaml
+++ b/test-network-k8s/kube/org0/org0-orderer3.yaml
@ -3,12 +3,13 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 #
+
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
  name: org0-orderer3-tls-cert
-  namespace: test-network
+  namespace: ${NS}
 spec:
  isCA: false
  privateKey:
@ -17,13 +18,15 @@ spec:
  dnsNames:
    - localhost
    - org0-orderer3
-    - org0-orderer3.test-network.svc.cluster.local
+    - org0-orderer3.${NS}.svc.cluster.local
+    - org0-orderer3.${DOMAIN}
+    - org0-orderer3-admin.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org0-orderer3-tls-cert
  issuerRef:
    name: org0-tls-cert-issuer
-    
+
 ---
 apiVersion: v1
 kind: ConfigMap
@ -41,6 +44,12 @@ data:
  ORDERER_GENERAL_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
  ORDERER_GENERAL_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
  ORDERER_GENERAL_BOOTSTRAPMETHOD: none
+  ORDERER_ADMIN_TLS_ENABLED: "true"
+  ORDERER_ADMIN_TLS_CERTIFICATE: /var/hyperledger/fabric/config/tls/tls.crt
+  ORDERER_ADMIN_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
+  ORDERER_ADMIN_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
+  # Authenticate client connections with the org's ecert / admin user enrollments
+  ORDERER_ADMIN_TLS_CLIENTROOTCAS: "[/var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/msp/cacerts/org0-ca.pem]"
  ORDERER_FILELEDGER_LOCATION: /var/hyperledger/fabric/data/orderer3
  ORDERER_CONSENSUS_WALDIR: /var/hyperledger/fabric/data/orderer3/etcdraft/wal
  ORDERER_CONSENSUS_SNAPDIR: /var/hyperledger/fabric/data/orderer3/etcdraft/wal
@ -64,7 +73,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-orderer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-orderer:${FABRIC_VERSION}
          imagePullPolicy: IfNotPresent
          envFrom:
            - configMapRef:
@ -108,4 +117,43 @@ spec:
      port: 9443
      protocol: TCP
  selector:
-    app: org0-orderer3
+    app: org0-orderer3
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org0-orderer3
+  name: org0-orderer3
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org0-orderer3.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer3
+                port:
+                  name: general
+            path: /
+            pathType: ImplementationSpecific
+    - host: org0-orderer3-admin.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer3
+                port:
+                  name: admin
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org0-orderer3.${DOMAIN}
+    - hosts:
+        - org0-orderer3-admin.${DOMAIN}
--- a/test-network-k8s/kube/org1/org1-admin-cli.yaml
+++ b/test-network-k8s/kube/org1/org1-admin-cli.yaml
@ -1,65 +0,0 @@
-#
-# Copyright IBM Corp. All Rights Reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: org1-admin-cli
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: org1-admin-cli
-  template:
-    metadata:
-      labels:
-        app: org1-admin-cli
-    spec:
-      containers:
-        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-tools:{{FABRIC_VERSION}}
-          imagePullPolicy: IfNotPresent
-          env:
-            - name: FABRIC_CFG_PATH
-              value: /var/hyperledger/fabric/config
-            - name: CORE_PEER_MSPCONFIGPATH
-              value: /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp
-            - name: CORE_PEER_TLS_ROOTCERT_FILE
-              value: /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/msp/tlscacerts/org1-tls-ca.pem
-          args:
-            - sleep
-            - "2147483647"
-          workingDir: /root
-          volumeMounts:
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-            - name: fabric-config
-              mountPath: /var/hyperledger/fabric/config
-
-      # This init container will unfurl all of the MSP archives listed in the msp-config config map.
-      initContainers:
-        - name: msp-unfurl
-          image: busybox
-          command:
-            - sh
-            - -c
-            - "for msp in $(ls /msp/msp-*.tgz); do echo $msp && tar zxvf $msp -C /var/hyperledger/fabric; done"
-          volumeMounts:
-            - name: msp-config
-              mountPath: /msp
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-
-      volumes:
-        - name: fabric-volume
-          persistentVolumeClaim:
-            claimName: fabric-org1
-        - name: fabric-config
-          configMap:
-            name: org1-config
-        - name: msp-config
-          configMap:
-            name: msp-config
--- a/test-network-k8s/kube/org1/org1-ca.yaml
+++ b/test-network-k8s/kube/org1/org1-ca.yaml
@ -17,6 +17,7 @@ spec:
    - localhost
    - org1-ca
    - org1-ca.test-network.svc.cluster.local
+    - org1-ca.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org1-ca-tls-cert
@ -40,7 +41,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-ca:{{FABRIC_CA_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-ca:${FABRIC_CA_VERSION}
          imagePullPolicy: IfNotPresent
          env:
            - name: FABRIC_CA_SERVER_CA_NAME
@ -88,8 +89,36 @@ metadata:
  name: org1-ca
 spec:
  ports:
-    - name: tls
+    - name: https
      port: 443
      protocol: TCP
  selector:
-    app: org1-ca
+    app: org1-ca
+    
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org1-ca
+  name: org1-ca
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org1-ca.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org1-ca
+                port:
+                  name: https
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org1-ca.${DOMAIN}
+  
--- a/test-network-k8s/kube/org1/org1-peer1.yaml
+++ b/test-network-k8s/kube/org1/org1-peer1.yaml
@ -17,8 +17,10 @@ spec:
  dnsNames:
    - localhost
    - org1-peer1
-    - org1-peer-gateway-svc
    - org1-peer1.test-network.svc.cluster.local
+    - org1-peer1.${DOMAIN}
+    - org1-peer-gateway-svc
+    - org1-peer-gateway-svc.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org1-peer1-tls-cert
@ -74,7 +76,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-peer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-peer:${FABRIC_VERSION}
          imagePullPolicy: IfNotPresent
          envFrom:
            - configMapRef:
@ -118,7 +120,7 @@ metadata:
  name: org1-peer1
 spec:
  ports:
-    - name: gossip
+    - name: grpc
      port: 7051
      protocol: TCP
    - name: chaincode
@ -136,8 +138,46 @@ metadata:
  name: org1-peer-gateway-svc
 spec:
  ports:
-    - name: gossip
+    - name: grpc
      port: 7051
      protocol: TCP
  selector:
-    org: org1
+    org: org1
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org1-peer1
+  name: org1-peer1
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org1-peer1.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org1-peer1
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+    - host: org1-peer-gateway-svc.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org1-peer1
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org1-peer1.${DOMAIN}
+    - hosts:
+        - org1-peer-gateway-svc.${DOMAIN}
--- a/test-network-k8s/kube/org1/org1-peer2.yaml
+++ b/test-network-k8s/kube/org1/org1-peer2.yaml
@ -19,6 +19,7 @@ spec:
    - org1-peer2
    - org1-peer-gateway-svc
    - org1-peer2.test-network.svc.cluster.local
+    - org1-peer2.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org1-peer2-tls-cert
@ -75,7 +76,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-peer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-peer:${FABRIC_VERSION}
          imagePullPolicy: IfNotPresent
          envFrom:
            - configMapRef:
@ -119,7 +120,7 @@ metadata:
  name: org1-peer2
 spec:
  ports:
-    - name: gossip
+    - name: grpc
      port: 7051
      protocol: TCP
    - name: chaincode
@ -129,4 +130,30 @@ spec:
      port: 9443
      protocol: TCP
  selector:
-    app: org1-peer2
+    app: org1-peer2
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org1-peer2
+  name: org1-peer2
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org1-peer2.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org1-peer2
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org1-peer2.${DOMAIN}
--- a/test-network-k8s/kube/org2/org2-admin-cli.yaml
+++ b/test-network-k8s/kube/org2/org2-admin-cli.yaml
@ -1,65 +0,0 @@
-#
-# Copyright IBM Corp. All Rights Reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: org2-admin-cli
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: org2-admin-cli
-  template:
-    metadata:
-      labels:
-        app: org2-admin-cli
-    spec:
-      containers:
-        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-tools:{{FABRIC_VERSION}}
-          imagePullPolicy: IfNotPresent
-          env:
-            - name: FABRIC_CFG_PATH
-              value: /var/hyperledger/fabric/config
-            - name: CORE_PEER_MSPCONFIGPATH
-              value: /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp
-            - name: CORE_PEER_TLS_ROOTCERT_FILE
-              value: /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/msp/tlscacerts/org2-tls-ca.pem
-          args:
-            - sleep
-            - "2147483647"
-          workingDir: /root
-          volumeMounts:
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-            - name: fabric-config
-              mountPath: /var/hyperledger/fabric/config
-
-      # This init container will unfurl all of the MSP archives listed in the msp-config config map.
-      initContainers:
-        - name: msp-unfurl
-          image: busybox
-          command:
-            - sh
-            - -c
-            - "for msp in $(ls /msp/msp-*.tgz); do echo $msp && tar zxvf $msp -C /var/hyperledger/fabric; done"
-          volumeMounts:
-            - name: msp-config
-              mountPath: /msp
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-
-      volumes:
-        - name: fabric-volume
-          persistentVolumeClaim:
-            claimName: fabric-org2
-        - name: fabric-config
-          configMap:
-            name: org2-config
-        - name: msp-config
-          configMap:
-            name: msp-config
--- a/test-network-k8s/kube/org2/org2-ca.yaml
+++ b/test-network-k8s/kube/org2/org2-ca.yaml
@ -17,6 +17,7 @@ spec:
    - localhost
    - org2-ca
    - org2-ca.test-network.svc.cluster.local
+    - org2-ca.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org2-ca-tls-cert
@ -40,7 +41,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-ca:{{FABRIC_CA_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-ca:${FABRIC_CA_VERSION}
          imagePullPolicy: IfNotPresent
          env:
            - name: FABRIC_CA_SERVER_CA_NAME
@ -88,8 +89,36 @@ metadata:
  name: org2-ca
 spec:
  ports:
-    - name: tls
+    - name: https
      port: 443
      protocol: TCP
  selector:
-    app: org2-ca
+    app: org2-ca
+    
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org2-ca
+  name: org2-ca
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org2-ca.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org2-ca
+                port:
+                  name: https
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org2-ca.${DOMAIN}
+    
--- a/test-network-k8s/kube/org2/org2-peer1.yaml
+++ b/test-network-k8s/kube/org2/org2-peer1.yaml
@ -17,8 +17,10 @@ spec:
  dnsNames:
    - localhost
    - org2-peer1
-    - org2-peer-gateway-svc
    - org2-peer1.test-network.svc.cluster.local
+    - org2-peer1.${DOMAIN}
+    - org2-peer-gateway-svc
+    - org2-peer-gateway-svc.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org2-peer1-tls-cert
@ -74,7 +76,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-peer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-peer:${FABRIC_VERSION}
          imagePullPolicy: IfNotPresent
          envFrom:
            - configMapRef:
@ -118,7 +120,7 @@ metadata:
  name: org2-peer1
 spec:
  ports:
-    - name: gossip
+    - name: grpc
      port: 7051
      protocol: TCP
    - name: chaincode
@ -136,8 +138,46 @@ metadata:
  name: org2-peer-gateway-svc
 spec:
  ports:
-    - name: gossip
+    - name: grpc
      port: 7051
      protocol: TCP
  selector:
    org: org2
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org2-peer1
+  name: org2-peer1
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org2-peer1.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org2-peer1
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+    - host: org2-peer-gateway-svc.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org2-peer1
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org2-peer1.${DOMAIN}
+    - hosts:
+        - org2-peer-gateway-svc.${DOMAIN}
--- a/test-network-k8s/kube/org2/org2-peer2.yaml
+++ b/test-network-k8s/kube/org2/org2-peer2.yaml
@ -19,6 +19,7 @@ spec:
    - org2-peer2
    - org2-peer-gateway-svc
    - org2-peer2.test-network.svc.cluster.local
+    - org2-peer2.${DOMAIN}
  ipAddresses:
    - 127.0.0.1
  secretName: org2-peer2-tls-cert
@ -74,7 +75,7 @@ spec:
    spec:
      containers:
        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-peer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-peer:${FABRIC_VERSION}
          imagePullPolicy: IfNotPresent
          envFrom:
            - configMapRef:
@ -118,7 +119,7 @@ metadata:
  name: org2-peer2
 spec:
  ports:
-    - name: gossip
+    - name: grpc
      port: 7051
      protocol: TCP
    - name: chaincode
@ -129,3 +130,29 @@ spec:
      protocol: TCP
  selector:
    app: org2-peer2
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org2-peer2
+  name: org2-peer2
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org2-peer2.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org2-peer2
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org2-peer2.${DOMAIN}
--- a/test-network-k8s/network
+++ b/test-network-k8s/network
@ -20,14 +20,17 @@ set -o errexit
 # todo: track down a nasty bug whereby the CA service endpoints (kube services) will occasionally reject TCP connections after network down/up.  This is patched by introducing a 10s sleep after the deployments are up...
 # todo: refactor query/invoke to specify chaincode name (-n param)

-CONTAINER_CLI=${CONTAINER_CLI:-docker}
-FABRIC_VERSION=${TEST_NETWORK_FABRIC_VERSION:-2.4}
-FABRIC_CA_VERSION=${TEST_NETWORK_FABRIC_CA_VERSION:-1.5}
-FABRIC_CONTAINER_REGISTRY=${TEST_NETWORK_FABRIC_CONTAINER_REGISTRY:-hyperledger}
-NETWORK_NAME=${TEST_NETWORK_NAME:-test-network}
-CLUSTER_NAME=${TEST_NETWORK_KIND_CLUSTER_NAME:-kind}
-NS=${TEST_NETWORK_KUBE_NAMESPACE:-${NETWORK_NAME}}
-CHANNEL_NAME=${TEST_NETWORK_CHANNEL_NAME:-mychannel}
+export CONTAINER_CLI=${CONTAINER_CLI:-docker}
+export FABRIC_VERSION=${TEST_NETWORK_FABRIC_VERSION:-2.4.3}
+export FABRIC_CA_VERSION=${TEST_NETWORK_FABRIC_CA_VERSION:-1.5.2}
+export FABRIC_CONTAINER_REGISTRY=${TEST_NETWORK_FABRIC_CONTAINER_REGISTRY:-hyperledger}
+export NETWORK_NAME=${TEST_NETWORK_NAME:-test-network}
+export CLUSTER_NAME=${TEST_NETWORK_KIND_CLUSTER_NAME:-kind}
+export NS=${TEST_NETWORK_KUBE_NAMESPACE:-${NETWORK_NAME}}
+export DOMAIN=${TEST_NETWORK_DOMAIN:-vcap.me}
+export CHANNEL_NAME=${TEST_NETWORK_CHANNEL_NAME:-mychannel}
+export TEMP_DIR=${PWD}/build
+
 LOG_FILE=${TEST_NETWORK_LOG_FILE:-network.log}
 DEBUG_FILE=${TEST_NETWORK_DEBUG_FILE:-network-debug.log}
 LOCAL_REGISTRY_NAME=${TEST_NETWORK_LOCAL_REGISTRY_NAME:-kind-registry}
@ -50,6 +53,7 @@ function print_help() {
  log "Fabric CA Version    \t: ${FABRIC_CA_VERSION}"
  log "Container Registry   \t: ${FABRIC_CONTAINER_REGISTRY}"
  log "Network name       \t\t: ${NETWORK_NAME}"
+  log "Ingress domain     \t\t: ${DOMAIN}"
  log "Channel name       \t\t: ${CHANNEL_NAME}"
  log
  log "--- Chaincode Information"
--- a/test-network-k8s/scripts/channel.sh
+++ b/test-network-k8s/scripts/channel.sh
@ -5,197 +5,272 @@
 # SPDX-License-Identifier: Apache-2.0
 #

-function create_channel_org_MSP() {
+function channel_up() {
+  set -x
+
+  register_org_admins
+  enroll_org_admins
+
+  create_channel_MSP
+  create_genesis_block
+
+  join_channel_orderers
+  join_channel_peers
+}
+
+function register_org_admins() {
+  push_fn "Registering org Admin users"
+
+  register_org_admin org0 org0admin org0adminpw
+  register_org_admin org1 org1admin org1adminpw
+  register_org_admin org2 org2admin org2adminpw
+
+  pop_fn
+}
+
+# Register the org admin user
+function register_org_admin() {
+  local type=admin
  local org=$1
-  local org_type=$2
-  local ecert_ca=${org}-ca
-  
-  echo 'set -x
- 
-  mkdir -p /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/cacerts
-  cp \
-    $FABRIC_CA_CLIENT_HOME/'${ecert_ca}'/rcaadmin/msp/cacerts/'${ecert_ca}'.pem \
-    /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/cacerts
+  local id_name=$2
+  local id_secret=$3
+  local ca_name=${org}-ca

-  mkdir -p /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/tlscacerts
-  cp \
-    /var/hyperledger/fabric/config/tls/ca.crt \
-    /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/tlscacerts/'${org}'-tls-ca.pem
+  echo "Registering org admin $username"

-  echo "NodeOUs:
-    Enable: true
-    ClientOUIdentifier:
-      Certificate: cacerts/'${ecert_ca}'.pem
-      OrganizationalUnitIdentifier: client
-    PeerOUIdentifier:
-      Certificate: cacerts/'${ecert_ca}'.pem
-      OrganizationalUnitIdentifier: peer
-    AdminOUIdentifier:
-      Certificate: cacerts/'${ecert_ca}'.pem
-      OrganizationalUnitIdentifier: admin
-    OrdererOUIdentifier:
-      Certificate: cacerts/'${ecert_ca}'.pem
-      OrganizationalUnitIdentifier: orderer "> /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/config.yaml
-      
-  ' | exec kubectl -n $NS exec deploy/${ecert_ca} -i -- /bin/sh
+  cat <<EOF | kubectl -n $NS exec deploy/${ca_name} -i -- /bin/sh
+
+  set -x
+  export FABRIC_CA_CLIENT_HOME=/var/hyperledger/fabric-ca-client
+  export FABRIC_CA_CLIENT_TLS_CERTFILES=/var/hyperledger/fabric/config/tls/ca.crt
+
+  fabric-ca-client register \
+    --id.name ${id_name} \
+    --id.secret ${id_secret} \
+    --id.type ${type} \
+    --url https://${ca_name} \
+    --mspdir /var/hyperledger/fabric-ca-client/${ca_name}/rcaadmin/msp \
+    --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
+EOF
+}
+
+function enroll_org_admins() {
+  push_fn "Enrolling org Admin users"
+
+  enroll_org_admin orderer  org0 org0admin org0adminpw
+  enroll_org_admin peer     org1 org1admin org1adminpw
+  enroll_org_admin peer     org2 org2admin org2adminpw
+
+  pop_fn
+}
+
+# Enroll the admin client to the local certificate storage folder.
+function enroll_org_admin() {
+  local type=$1
+  local org=$2
+  local username=$3
+  local password=$4
+
+  echo "Enrolling $type org admin $username"
+
+  ENROLLMENTS_DIR=${TEMP_DIR}/enrollments
+  ORG_ADMIN_DIR=${ENROLLMENTS_DIR}/${org}/users/${username}
+
+  # skip the enrollment if the admin certificate is available.
+  if [ -f "${ORG_ADMIN_DIR}/msp/keystore/key.pem" ]; then
+    echo "Found an existing admin enrollment at ${ORG_ADMIN_DIR}"
+    return
+  fi
+
+  # Determine the CA information and TLS certificate
+  CA_NAME=${org}-ca
+  CA_DIR=${TEMP_DIR}/cas/${CA_NAME}
+  mkdir -p ${CA_DIR}
+
+  CA_AUTH=${username}:${password}
+  CA_HOST=${CA_NAME}.${DOMAIN}
+  CA_PORT=443
+  CA_URL=https://${CA_AUTH}@${CA_HOST}:${CA_PORT}
+
+  # Read the CA's TLS certificate from the cert-manager CA secret
+  echo "retrieving ${org}-ca TLS cert"
+  kubectl -n $NS get secret ${org}-ca-tls-cert -o json \
+    | jq -r .data.\"tls.crt\" \
+    | base64 -d \
+    > ${CA_DIR}/tls-cert.pem
+
+  # enroll the org admin
+  FABRIC_CA_CLIENT_HOME=${ORG_ADMIN_DIR} fabric-ca-client enroll \
+    --url ${CA_URL} \
+    --tls.certfiles ${CA_DIR}/tls-cert.pem
+
+
+  # Construct an msp config.yaml
+  CA_CERT_NAME=${CA_NAME}-$(echo $DOMAIN | tr -s . -)-${CA_PORT}.pem
+
+  create_msp_config_yaml ${CA_NAME} ${CA_CERT_NAME} ${ORG_ADMIN_DIR}/msp
+
+  # private keys are hashed by name, but we only support one enrollment.
+  # test-network examples refer to this as "server.key", which is incorrect.
+  # This is the private key used to endorse transactions using the admin's
+  # public key.
+  mv ${ORG_ADMIN_DIR}/msp/keystore/*_sk ${ORG_ADMIN_DIR}/msp/keystore/key.pem
+}
+
+# create an enrollment MSP config.yaml
+function create_msp_config_yaml() {
+  local ca_name=$1
+  local ca_cert_name=$2
+  local msp_dir=$3
+  echo "Creating msp config ${msp_dir}/config.yaml with cert ${ca_cert_name}"
+
+  cat << EOF > ${msp_dir}/config.yaml
+NodeOUs:
+  Enable: true
+  ClientOUIdentifier:
+    Certificate: cacerts/${ca_cert_name}
+    OrganizationalUnitIdentifier: client
+  PeerOUIdentifier:
+    Certificate: cacerts/${ca_cert_name}
+    OrganizationalUnitIdentifier: peer
+  AdminOUIdentifier:
+    Certificate: cacerts/${ca_cert_name}
+    OrganizationalUnitIdentifier: admin
+  OrdererOUIdentifier:
+    Certificate: cacerts/${ca_cert_name}
+    OrganizationalUnitIdentifier: orderer
+EOF
 }

 function create_channel_MSP() {
  push_fn "Creating channel MSP"

-  create_channel_org_MSP org0 orderer 
+  create_channel_org_MSP org0 orderer
  create_channel_org_MSP org1 peer
  create_channel_org_MSP org2 peer

-  pop_fn
-}
-
-function aggregate_channel_MSP() {
-  push_fn "Aggregating channel MSP"
-
-  rm -rf ./build/msp/
-  mkdir -p ./build/msp
-  
-  kubectl -n $NS exec deploy/org0-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/ordererOrganizations/org0.example.com/msp > build/msp/msp-org0.example.com.tgz
-  kubectl -n $NS exec deploy/org1-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/peerOrganizations/org1.example.com/msp > build/msp/msp-org1.example.com.tgz
-  kubectl -n $NS exec deploy/org2-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/peerOrganizations/org2.example.com/msp > build/msp/msp-org2.example.com.tgz
-
-  kubectl -n $NS delete configmap msp-config || true
-  kubectl -n $NS create configmap msp-config --from-file=build/msp/
+  extract_orderer_tls_cert org0 orderer1
+  extract_orderer_tls_cert org0 orderer2
+  extract_orderer_tls_cert org0 orderer3

  pop_fn
 }

-function launch_admin_CLIs() {
-  push_fn "Launching admin CLIs"
+function create_channel_org_MSP() {
+  local org=$1
+  local type=$2
+  local ca_name=${org}-ca

-  launch kube/org0/org0-admin-cli.yaml
-  launch kube/org1/org1-admin-cli.yaml
-  launch kube/org2/org2-admin-cli.yaml
+  ORG_MSP_DIR=${TEMP_DIR}/channel-msp/${type}Organizations/${org}/msp
+  mkdir -p ${ORG_MSP_DIR}/cacerts
+  mkdir -p ${ORG_MSP_DIR}/tlscacerts

-  kubectl -n $NS rollout status deploy/org0-admin-cli
-  kubectl -n $NS rollout status deploy/org1-admin-cli
-  kubectl -n $NS rollout status deploy/org2-admin-cli
+  # extract the CA's signing authority from the CA/cainfo response
+  curl -s \
+    --cacert ${TEMP_DIR}/cas/${ca_name}/tls-cert.pem \
+    https://${ca_name}.${DOMAIN}/cainfo \
+    | jq -r .result.CAChain \
+    | base64 -d \
+    > ${ORG_MSP_DIR}/cacerts/ca-signcert.pem

-  pop_fn
+  # extract the CA's TLS CA certificate from the cert-manager secret
+  kubectl -n $NS get secret ${ca_name}-tls-cert -o json \
+    | jq -r .data.\"ca.crt\" \
+    | base64 -d \
+    > ${ORG_MSP_DIR}/tlscacerts/tlsca-signcert.pem
+
+  # create an MSP config.yaml with the CA's signing certificate
+  create_msp_config_yaml ${ca_name} ca-signcert.pem ${ORG_MSP_DIR}
+}
+
+# Extract an orderer's TLS signing certificate for inclusion in the channel config block
+function extract_orderer_tls_cert() {
+  local org=$1
+  local orderer=$2
+
+  echo "Extracting TLS cert for $org $orderer"
+
+  ORDERER_TLS_DIR=${TEMP_DIR}/channel-msp/ordererOrganizations/${org}/orderers/${org}-${orderer}/tls
+  mkdir -p $ORDERER_TLS_DIR/signcerts
+
+  kubectl -n $NS get secret ${org}-${orderer}-tls-cert -o json \
+    | jq -r .data.\"tls.crt\" \
+    | base64 -d \
+    > ${ORDERER_TLS_DIR}/signcerts/tls-cert.pem
 }

 function create_genesis_block() {
-  push_fn "Creating channel \"${CHANNEL_NAME}\""
+  push_fn "Creating channel genesis block"

-  echo 'set -x
-  configtxgen -profile TwoOrgsApplicationGenesis -channelID '${CHANNEL_NAME}' -outputBlock genesis_block.pb
-  # configtxgen -inspectBlock genesis_block.pb
-  
-  osnadmin channel join --orderer-address org0-orderer1:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb
-  osnadmin channel join --orderer-address org0-orderer2:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb
-  osnadmin channel join --orderer-address org0-orderer3:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb
-  
-  ' | exec kubectl -n $NS exec deploy/org0-admin-cli -i -- /bin/bash
-  
-  # todo: readiness / liveiness equivalent for channel ?    Needs a little bit to settle before peers can join. 
+  FABRIC_CFG_PATH=${PWD}/config/org0 \
+    configtxgen \
+      -profile TwoOrgsApplicationGenesis \
+      -channelID $CHANNEL_NAME \
+      -outputBlock ${TEMP_DIR}/genesis_block.pb
+
+  # configtxgen -inspectBlock ${TEMP_DIR}/genesis_block.pb
+
+  pop_fn
+}
+
+function join_channel_orderers() {
+  push_fn "Joining orderers to channel ${CHANNEL_NAME}"
+
+  join_channel_orderer org0 orderer1
+  join_channel_orderer org0 orderer2
+  join_channel_orderer org0 orderer3
+
+  # todo: readiness / liveiness equivalent for channel?  Needs a little bit to settle before peers can join.
  sleep 10

  pop_fn
 }

-function join_org_peers() {
+# Request from the channel ADMIN api that the orderer joins the target channel
+function join_channel_orderer() {
  local org=$1
-  push_fn "Joining ${org} peers to channel \"${CHANNEL_NAME}\""
+  local orderer=$2

-  echo 'set -x
-  # Fetch the genesis block from an orderer
-  peer channel \
-    fetch oldest \
-    genesis_block.pb \
-    -c '${CHANNEL_NAME}' \
-    -o org0-orderer1:6050 \
-    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
-
-  # Join peer1 to the channel.
-  CORE_PEER_ADDRESS='${org}'-peer1:7051 \
-  peer channel \
-    join \
-    -b genesis_block.pb \
-    -o org0-orderer1:6050 \
-    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
-
-  # Join peer2 to the channel.
-  CORE_PEER_ADDRESS='${org}'-peer2:7051 \
-  peer channel \
-    join \
-    -b genesis_block.pb \
-    -o org0-orderer1:6050 \
-    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
-
-  ' | exec kubectl -n $NS exec deploy/${org}-admin-cli -i -- /bin/bash
-
-  pop_fn
+  # The client certificate presented in this case is the admin user's enrollment key.  This is a stronger assertion
+  # of identity than the Docker Compose network, which transmits the orderer node's TLS key pair directly
+  osnadmin channel join \
+    --orderer-address ${org}-${orderer}-admin.${DOMAIN} \
+    --ca-file ${TEMP_DIR}/channel-msp/ordererOrganizations/${org}/orderers/${org}-${orderer}/tls/signcerts/tls-cert.pem \
+    --client-cert ${TEMP_DIR}/enrollments/${org}/users/${org}admin/msp/signcerts/cert.pem \
+    --client-key ${TEMP_DIR}/enrollments/${org}/users/${org}admin/msp/keystore/key.pem \
+    --channelID ${CHANNEL_NAME} \
+    --config-block ${TEMP_DIR}/genesis_block.pb
 }

-function join_peers() {
+function join_channel_peers() {
  join_org_peers org1
  join_org_peers org2
 }

-# Copy the scripts/anchor_peers.sh to a remote volume
-function push_anchor_peer_script() {
+function join_org_peers() {
  local org=$1
+  push_fn "Joining ${org} peers to channel ${CHANNEL_NAME}"

-  tar cf - scripts/ | kubectl -n $NS exec -i -c main deploy/${org}-admin-cli -- tar xf - -C /var/hyperledger/fabric
-}
+  # fetch the genesis block from an orderer.   (do we need to do this?  It's available locally...)

-verify_result() {
-  if [ $1 -ne 0 ]; then
-    echo $2
-    exit $1
-  fi
-}
-
-# Launch the anchor peer update script on a remote org admin CLI
-function invoke_anchor_peer_update() {
-  local org_num=$1
-  local peer_name=$2
-
-  kubectl exec \
-    -n $NS \
-    -c main \
-    deploy/org${org_num}-admin-cli \
-    -i \
-    /bin/bash -c "/var/hyperledger/fabric/scripts/set_anchor_peer.sh ${org_num} ${CHANNEL_NAME} ${peer_name}"
-
-  verify_result $? "Error updating anchor peer for org ${org_num}"
-}
-
-#
-# To update the anchor peers we will need to execute a script on each of the peer admin CLI containers.  These
-# commands can be individually piped into kubectl exec ... but it will be simpler if we transfer the anchor
-# peer update script over to the org volume and then trigger it from kubectl.
-#
-function update_anchor_peers() {
-  local peer_name=$1
-  push_fn "Updating anchor peers to ${peer_name}"
-
-  push_anchor_peer_script org1
-  push_anchor_peer_script org2
-
-  invoke_anchor_peer_update 1 ${peer_name}
-  invoke_anchor_peer_update 2 ${peer_name}
+  # Join peers to channel
+  join_channel_peer $org peer1
+  join_channel_peer $org peer2

  pop_fn
 }

-function channel_up() {
+function join_channel_peer() {
+  local org=$1
+  local peer=$2

-  create_channel_MSP
-  aggregate_channel_MSP
-  launch_admin_CLIs
-
-  create_genesis_block
-  join_peers
-
-  # peer1 was set as the anchor peer in configtx.yaml.  Setting this again will force an
-  # error to be returned from the channel up.  We might want to render the warning in
-  # this case to indicate that the call was made but had a nonzero exit. 
-  # update_anchor_peers peer1
-}
+  FABRIC_CFG_PATH=${PWD}/config/${org} \
+  CORE_PEER_ADDRESS=${org}-${peer}.${DOMAIN}:443 \
+  CORE_PEER_MSPCONFIGPATH=${TEMP_DIR}/enrollments/${org}/users/${org}admin/msp \
+  CORE_PEER_TLS_ROOTCERT_FILE=${TEMP_DIR}/channel-msp/peerOrganizations/${org}/msp/tlscacerts/tlsca-signcert.pem \
+    peer channel join \
+      --blockpath ${TEMP_DIR}/genesis_block.pb \
+      --orderer   org0-orderer1.${DOMAIN} \
+      --tls        \
+      --cafile    ${TEMP_DIR}/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem
+}
--- a/test-network-k8s/scripts/fabric_CAs.sh
+++ b/test-network-k8s/scripts/fabric_CAs.sh
@ -5,20 +5,12 @@
 # SPDX-License-Identifier: Apache-2.0
 #

-function launch_CA() {
-  local yaml=$1
-  cat ${yaml} \
-    | sed 's,{{FABRIC_CONTAINER_REGISTRY}},'${FABRIC_CONTAINER_REGISTRY}',g' \
-    | sed 's,{{FABRIC_CA_VERSION}},'${FABRIC_CA_VERSION}',g' \
-    | kubectl -n $NS apply -f -
-}
-
 function launch_ECert_CAs() {
  push_fn "Launching Fabric CAs"

-  launch_CA kube/org0/org0-ca.yaml
-  launch_CA kube/org1/org1-ca.yaml
-  launch_CA kube/org2/org2-ca.yaml
+  apply_template kube/org0/org0-ca.yaml
+  apply_template kube/org1/org1-ca.yaml
+  apply_template kube/org2/org2-ca.yaml

  kubectl -n $NS rollout status deploy/org0-ca
  kubectl -n $NS rollout status deploy/org1-ca
--- a/test-network-k8s/scripts/kind.sh
+++ b/test-network-k8s/scripts/kind.sh
@ -87,8 +87,8 @@ nodes:
      - containerPort: 443
        hostPort: ${ingress_https_port}
        protocol: TCP
-networking:
-  kubeProxyMode: "ipvs"
+#networking:
+#  kubeProxyMode: "ipvs"

 # create a cluster with the local registry enabled in containerd
 containerdConfigPatches:
@ -98,6 +98,12 @@ containerdConfigPatches:

 EOF

+  # workaround for https://github.com/hyperledger/fabric-samples/issues/550 - pods can not resolve external DNS
+  for node in $(kind get nodes);
+  do
+    docker exec "$node" sysctl net.ipv4.conf.all.route_localnet=1;
+  done
+
  pop_fn
 }

--- a/test-network-k8s/scripts/old_channel.sh
+++ b/test-network-k8s/scripts/old_channel.sh
@ -0,0 +1,186 @@
+#!/bin/bash
+#
+# Copyright IBM Corp All Rights Reserved
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+function create_channel_org_MSP() {
+  local org=$1
+  local org_type=$2
+  local ecert_ca=${org}-ca
+  
+  echo 'set -x
+ 
+  mkdir -p /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/cacerts
+  cp \
+    $FABRIC_CA_CLIENT_HOME/'${ecert_ca}'/rcaadmin/msp/cacerts/'${ecert_ca}'.pem \
+    /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/cacerts
+
+  mkdir -p /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/tlscacerts
+  cp \
+    /var/hyperledger/fabric/config/tls/ca.crt \
+    /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/tlscacerts/'${org}'-tls-ca.pem
+
+  echo "NodeOUs:
+    Enable: true
+    ClientOUIdentifier:
+      Certificate: cacerts/'${ecert_ca}'.pem
+      OrganizationalUnitIdentifier: client
+    PeerOUIdentifier:
+      Certificate: cacerts/'${ecert_ca}'.pem
+      OrganizationalUnitIdentifier: peer
+    AdminOUIdentifier:
+      Certificate: cacerts/'${ecert_ca}'.pem
+      OrganizationalUnitIdentifier: admin
+    OrdererOUIdentifier:
+      Certificate: cacerts/'${ecert_ca}'.pem
+      OrganizationalUnitIdentifier: orderer "> /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/config.yaml
+      
+  ' | exec kubectl -n $NS exec deploy/${ecert_ca} -i -- /bin/sh
+}
+
+function create_channel_MSP() {
+  push_fn "Creating channel MSP"
+
+  create_channel_org_MSP org0 orderer 
+  create_channel_org_MSP org1 peer
+  create_channel_org_MSP org2 peer
+
+  pop_fn
+}
+
+function aggregate_channel_MSP() {
+  push_fn "Aggregating channel MSP"
+
+  rm -rf ./build/msp/
+  mkdir -p ./build/msp
+  
+  kubectl -n $NS exec deploy/org0-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/ordererOrganizations/org0.example.com/msp > build/msp/msp-org0.example.com.tgz
+  kubectl -n $NS exec deploy/org1-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/peerOrganizations/org1.example.com/msp > build/msp/msp-org1.example.com.tgz
+  kubectl -n $NS exec deploy/org2-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/peerOrganizations/org2.example.com/msp > build/msp/msp-org2.example.com.tgz
+
+  kubectl -n $NS delete configmap msp-config || true
+  kubectl -n $NS create configmap msp-config --from-file=build/msp/
+
+  pop_fn
+}
+
+function create_genesis_block() {
+  push_fn "Creating channel \"${CHANNEL_NAME}\""
+
+  echo 'set -x
+  configtxgen -profile TwoOrgsApplicationGenesis -channelID '${CHANNEL_NAME}' -outputBlock genesis_block.pb
+  # configtxgen -inspectBlock genesis_block.pb
+  
+  osnadmin channel join --orderer-address org0-orderer1:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb
+  osnadmin channel join --orderer-address org0-orderer2:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb
+  osnadmin channel join --orderer-address org0-orderer3:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb
+  
+  ' | exec kubectl -n $NS exec deploy/org0-admin-cli -i -- /bin/bash
+  
+  # todo: readiness / liveiness equivalent for channel ?    Needs a little bit to settle before peers can join. 
+  sleep 10
+
+  pop_fn
+}
+
+function join_org_peers() {
+  local org=$1
+  push_fn "Joining ${org} peers to channel \"${CHANNEL_NAME}\""
+
+  echo 'set -x
+  # Fetch the genesis block from an orderer
+  peer channel \
+    fetch oldest \
+    genesis_block.pb \
+    -c '${CHANNEL_NAME}' \
+    -o org0-orderer1:6050 \
+    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
+
+  # Join peer1 to the channel.
+  CORE_PEER_ADDRESS='${org}'-peer1:7051 \
+  peer channel \
+    join \
+    -b genesis_block.pb \
+    -o org0-orderer1:6050 \
+    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
+
+  # Join peer2 to the channel.
+  CORE_PEER_ADDRESS='${org}'-peer2:7051 \
+  peer channel \
+    join \
+    -b genesis_block.pb \
+    -o org0-orderer1:6050 \
+    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
+
+  ' | exec kubectl -n $NS exec deploy/${org}-admin-cli -i -- /bin/bash
+
+  pop_fn
+}
+
+function join_peers() {
+  join_org_peers org1
+  join_org_peers org2
+}
+
+# Copy the scripts/anchor_peers.sh to a remote volume
+function push_anchor_peer_script() {
+  local org=$1
+
+  tar cf - scripts/ | kubectl -n $NS exec -i -c main deploy/${org}-admin-cli -- tar xf - -C /var/hyperledger/fabric
+}
+
+verify_result() {
+  if [ $1 -ne 0 ]; then
+    echo $2
+    exit $1
+  fi
+}
+
+# Launch the anchor peer update script on a remote org admin CLI
+function invoke_anchor_peer_update() {
+  local org_num=$1
+  local peer_name=$2
+
+  kubectl exec \
+    -n $NS \
+    -c main \
+    deploy/org${org_num}-admin-cli \
+    -i \
+    /bin/bash -c "/var/hyperledger/fabric/scripts/set_anchor_peer.sh ${org_num} ${CHANNEL_NAME} ${peer_name}"
+
+  verify_result $? "Error updating anchor peer for org ${org_num}"
+}
+
+#
+# To update the anchor peers we will need to execute a script on each of the peer admin CLI containers.  These
+# commands can be individually piped into kubectl exec ... but it will be simpler if we transfer the anchor
+# peer update script over to the org volume and then trigger it from kubectl.
+#
+function update_anchor_peers() {
+  local peer_name=$1
+  push_fn "Updating anchor peers to ${peer_name}"
+
+  push_anchor_peer_script org1
+  push_anchor_peer_script org2
+
+  invoke_anchor_peer_update 1 ${peer_name}
+  invoke_anchor_peer_update 2 ${peer_name}
+
+  pop_fn
+}
+
+function channel_up() {
+
+  create_channel_MSP
+  aggregate_channel_MSP
+
+  create_genesis_block
+  join_peers
+
+  # peer1 was set as the anchor peer in configtx.yaml.  Setting this again will force an
+  # error to be returned from the channel up.  We might want to render the warning in
+  # this case to indicate that the call was made but had a nonzero exit. 
+  # update_anchor_peers peer1
+}
--- a/test-network-k8s/scripts/prereqs.sh
+++ b/test-network-k8s/scripts/prereqs.sh
@ -8,6 +8,8 @@
 # Double check that kind, kubectl, docker, and all required images are present.
 function check_prereqs() {

+  set +e
+
  ${CONTAINER_CLI} version > /dev/null
  if [[ $? -ne 0 ]]; then
    echo "No '${CONTAINER_CLI}' binary available?"
@ -31,4 +33,27 @@ function check_prereqs() {
    echo "No 'jq' binary available? (https://stedolan.github.io/jq/)"
    exit 1
  fi
+
+  # Use the local fabric binaries if available.  If not, go get them.
+  bin/peer version &> /dev/null
+  if [[ $? -ne 0 ]]; then
+    echo "Downloading LATEST Fabric binaries and config"
+    curl -sSL https://raw.githubusercontent.com/hyperledger/fabric/main/scripts/bootstrap.sh | bash -s -- -s -d
+
+    # remove sample config files extracted by the installation script
+    rm config/configtx.yaml
+    rm config/core.yaml
+    rm config/orderer.yaml
+  fi
+
+  export PATH=bin:$PATH
+
+  # Double-check that the binary transfer was OK
+  peer version > /dev/null
+  if [[ $? -ne 0 ]]; then
+    log "No 'peer' binary available?"
+    exit 1
+  fi
+
+  set -e
 }
--- a/test-network-k8s/scripts/test_network.sh
+++ b/test-network-k8s/scripts/test_network.sh
@ -5,20 +5,12 @@
 # SPDX-License-Identifier: Apache-2.0
 #

-function launch() {
-  local yaml=$1
-  cat ${yaml} \
-    | sed 's,{{FABRIC_CONTAINER_REGISTRY}},'${FABRIC_CONTAINER_REGISTRY}',g' \
-    | sed 's,{{FABRIC_VERSION}},'${FABRIC_VERSION}',g' \
-    | kubectl -n $NS apply -f -
-}
-
 function launch_orderers() {
  push_fn "Launching orderers"

-  launch kube/org0/org0-orderer1.yaml
-  launch kube/org0/org0-orderer2.yaml
-  launch kube/org0/org0-orderer3.yaml
+  apply_template kube/org0/org0-orderer1.yaml
+  apply_template kube/org0/org0-orderer2.yaml
+  apply_template kube/org0/org0-orderer3.yaml

  kubectl -n $NS rollout status deploy/org0-orderer1
  kubectl -n $NS rollout status deploy/org0-orderer2
@ -30,10 +22,10 @@ function launch_orderers() {
 function launch_peers() {
  push_fn "Launching peers"

-  launch kube/org1/org1-peer1.yaml
-  launch kube/org1/org1-peer2.yaml
-  launch kube/org2/org2-peer1.yaml
-  launch kube/org2/org2-peer2.yaml
+  apply_template kube/org1/org1-peer1.yaml
+  apply_template kube/org1/org1-peer2.yaml
+  apply_template kube/org2/org2-peer1.yaml
+  apply_template kube/org2/org2-peer2.yaml

  kubectl -n $NS rollout status deploy/org1-peer1
  kubectl -n $NS rollout status deploy/org1-peer2
@ -43,155 +35,127 @@ function launch_peers() {
  pop_fn
 }

-function create_org0_local_MSP() {
-  echo 'set -x
+# todo: enroll org admin LOCALLY from the host OS
+#  fabric-ca-client register --id.name org1-admin --id.secret org1adminpw  --id.type admin   --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
+#  fabric-ca-client enroll --url https://org1-admin:org1adminpw@org1-ca  --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/server.key
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/config.yaml
+
+#  fabric-ca-client register --id.name org1-admin --id.secret org1adminpw  --id.type admin   --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
+#  fabric-ca-client enroll --url https://org1-admin:org1adminpw@org1-ca  --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/server.key
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/config.yaml
+
+#  fabric-ca-client register --id.name org2-admin --id.secret org2adminpw  --id.type admin   --url https://org2-ca --mspdir $FABRIC_CA_CLIENT_HOME/org2-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
+#  fabric-ca-client enroll --url https://org2-admin:org2adminpw@org2-ca  --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/keystore/server.key
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/config.yaml
+
+
+# Each network node needs a registration, enrollment, and MSP config.yaml
+function create_node_local_MSP() {
+  local node_type=$1
+  local org=$2
+  local node=$3
+  local csr_hosts=$4
+  local id_name=${org}-${node}
+  local id_secret=${node_type}pw
+  local ca_name=${org}-ca
+
+  cat <<EOF | kubectl -n $NS exec deploy/${ca_name} -i -- /bin/sh
+
+  set -x
  export FABRIC_CA_CLIENT_HOME=/var/hyperledger/fabric-ca-client
  export FABRIC_CA_CLIENT_TLS_CERTFILES=/var/hyperledger/fabric/config/tls/ca.crt

  # Each identity in the network needs a registration and enrollment.
-  fabric-ca-client register --id.name org0-orderer1 --id.secret ordererpw --id.type orderer --url https://org0-ca --mspdir $FABRIC_CA_CLIENT_HOME/org0-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org0-orderer2 --id.secret ordererpw --id.type orderer --url https://org0-ca --mspdir $FABRIC_CA_CLIENT_HOME/org0-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org0-orderer3 --id.secret ordererpw --id.type orderer --url https://org0-ca --mspdir $FABRIC_CA_CLIENT_HOME/org0-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org0-admin --id.secret org0adminpw  --id.type admin   --url https://org0-ca --mspdir $FABRIC_CA_CLIENT_HOME/org0-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
+  fabric-ca-client register \
+    --id.name ${id_name} \
+    --id.secret ${id_secret} \
+    --id.type ${node_type} \
+    --url https://${ca_name} \
+    --mspdir /var/hyperledger/fabric-ca-client/${ca_name}/rcaadmin/msp

-  fabric-ca-client enroll --url https://org0-orderer1:ordererpw@org0-ca --csr.hosts org0-orderer1 --mspdir /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp
-  fabric-ca-client enroll --url https://org0-orderer2:ordererpw@org0-ca --csr.hosts org0-orderer2 --mspdir /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/msp
-  fabric-ca-client enroll --url https://org0-orderer3:ordererpw@org0-ca --csr.hosts org0-orderer3 --mspdir /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/msp
-  fabric-ca-client enroll --url https://org0-admin:org0adminpw@org0-ca --mspdir /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/users/Admin@org0.example.com/msp
-
-  # Create an MSP config.yaml (why is this not generated by the enrollment by fabric-ca-client?)
-  echo "NodeOUs:
-    Enable: true
-    ClientOUIdentifier:
-      Certificate: cacerts/org0-ca.pem
-      OrganizationalUnitIdentifier: client
-    PeerOUIdentifier:
-      Certificate: cacerts/org0-ca.pem
-      OrganizationalUnitIdentifier: peer
-    AdminOUIdentifier:
-      Certificate: cacerts/org0-ca.pem
-      OrganizationalUnitIdentifier: admin
-    OrdererOUIdentifier:
-      Certificate: cacerts/org0-ca.pem
-      OrganizationalUnitIdentifier: orderer" > /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp/config.yaml
-
-  cp /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/msp/config.yaml
-  cp /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/msp/config.yaml
-  ' | exec kubectl -n $NS exec deploy/org0-ca -i -- /bin/sh
-}
-
-function create_org1_local_MSP() {
-
-  echo 'set -x
-  export FABRIC_CA_CLIENT_HOME=/var/hyperledger/fabric-ca-client
-  export FABRIC_CA_CLIENT_TLS_CERTFILES=/var/hyperledger/fabric/config/tls/ca.crt
-
-  # Each identity in the network needs a registration and enrollment.
-  fabric-ca-client register --id.name org1-peer1 --id.secret peerpw --id.type peer --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org1-peer2 --id.secret peerpw --id.type peer --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org1-admin --id.secret org1adminpw  --id.type admin   --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
-
-  fabric-ca-client enroll --url https://org1-peer1:peerpw@org1-ca --csr.hosts localhost,org1-peer1,org1-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp
-  fabric-ca-client enroll --url https://org1-peer2:peerpw@org1-ca --csr.hosts localhost,org1-peer2,org1-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer2.org1.example.com/msp
-  fabric-ca-client enroll --url https://org1-admin:org1adminpw@org1-ca  --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp
-
-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/server.key
+  fabric-ca-client enroll \
+    --url https://${id_name}:${id_secret}@${ca_name} \
+    --csr.hosts ${csr_hosts} \
+    --mspdir /var/hyperledger/fabric/organizations/${node_type}Organizations/${org}.example.com/${node_type}s/${id_name}.${org}.example.com/msp

  # Create local MSP config.yaml
  echo "NodeOUs:
    Enable: true
    ClientOUIdentifier:
-      Certificate: cacerts/org1-ca.pem
+      Certificate: cacerts/${org}-ca.pem
      OrganizationalUnitIdentifier: client
    PeerOUIdentifier:
-      Certificate: cacerts/org1-ca.pem
+      Certificate: cacerts/${org}-ca.pem
      OrganizationalUnitIdentifier: peer
    AdminOUIdentifier:
-      Certificate: cacerts/org1-ca.pem
+      Certificate: cacerts/${org}-ca.pem
      OrganizationalUnitIdentifier: admin
    OrdererOUIdentifier:
-      Certificate: cacerts/org1-ca.pem
-      OrganizationalUnitIdentifier: orderer" > /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml
-
-
-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer2.org1.example.com/msp/config.yaml
-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/config.yaml
-  ' | exec kubectl -n $NS exec deploy/org1-ca -i -- /bin/sh
-
+      Certificate: cacerts/${org}-ca.pem
+      OrganizationalUnitIdentifier: orderer" > /var/hyperledger/fabric/organizations/${node_type}Organizations/${org}.example.com/${node_type}s/${id_name}.${org}.example.com/msp/config.yaml
+EOF
 }

-function create_org2_local_MSP() {
-  echo 'set -x
-  export FABRIC_CA_CLIENT_HOME=/var/hyperledger/fabric-ca-client
-  export FABRIC_CA_CLIENT_TLS_CERTFILES=/var/hyperledger/fabric/config/tls/ca.crt
+function create_orderer_local_MSP() {
+  local org=$1
+  local orderer=$2
+  local csr_hosts=${org}-${orderer}

-  # Each identity in the network needs a registration and enrollment.
-  fabric-ca-client register --id.name org2-peer1 --id.secret peerpw --id.type peer --url https://org2-ca --mspdir $FABRIC_CA_CLIENT_HOME/org2-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org2-peer2 --id.secret peerpw --id.type peer --url https://org2-ca --mspdir $FABRIC_CA_CLIENT_HOME/org2-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org2-admin --id.secret org2adminpw  --id.type admin   --url https://org2-ca --mspdir $FABRIC_CA_CLIENT_HOME/org2-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
+  create_node_local_MSP orderer $org $orderer $csr_hosts
+}

-  fabric-ca-client enroll --url https://org2-peer1:peerpw@org2-ca --csr.hosts localhost,org2-peer1,org2-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp
-  fabric-ca-client enroll --url https://org2-peer2:peerpw@org2-ca --csr.hosts localhost,org2-peer2,org2-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer2.org2.example.com/msp
-  fabric-ca-client enroll --url https://org2-admin:org2adminpw@org2-ca  --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp
+function create_peer_local_MSP() {
+  local org=$1
+  local peer=$2
+  local csr_hosts=localhost,${org}-${peer},${org}-peer-gateway-svc

-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/keystore/server.key
-
-  # Create local MSP config.yaml
-  echo "NodeOUs:
-    Enable: true
-    ClientOUIdentifier:
-      Certificate: cacerts/org2-ca.pem
-      OrganizationalUnitIdentifier: client
-    PeerOUIdentifier:
-      Certificate: cacerts/org2-ca.pem
-      OrganizationalUnitIdentifier: peer
-    AdminOUIdentifier:
-      Certificate: cacerts/org2-ca.pem
-      OrganizationalUnitIdentifier: admin
-    OrdererOUIdentifier:
-      Certificate: cacerts/org2-ca.pem
-      OrganizationalUnitIdentifier: orderer" > /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp/config.yaml
-
-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer2.org2.example.com/msp/config.yaml
-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/config.yaml
-  ' | exec kubectl -n $NS exec deploy/org2-ca -i -- /bin/sh
+  create_node_local_MSP peer $org $peer $csr_hosts
 }

 function create_local_MSP() {
  push_fn "Creating local node MSP"

-  create_org0_local_MSP
-  create_org1_local_MSP
-  create_org2_local_MSP
-
-  pop_fn
-}
-
-# TLS certificates are isused by the CA's Issuer, stored in a Kube secret, and mounted into the pod at /var/hyperledger/fabric/config/tls.
-# For consistency with the Fabric-CA guide, his function copies the orderer's TLS certs into the traditional Fabric MSP / folder structure.
-function extract_orderer_tls_cert() {
-  local orderer=$1
-
-  echo 'set -x
-
-  mkdir -p /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/'${orderer}'.org0.example.com/tls/signcerts/
-
-  cp \
-    var/hyperledger/fabric/config/tls/tls.crt \
-    /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/'${orderer}'.org0.example.com/tls/signcerts/cert.pem
-
-  ' | exec kubectl -n $NS exec deploy/${orderer} -i -c main -- /bin/sh
-}
-
-function extract_orderer_tls_certs() {
-  push_fn "Extracting orderer TLS certs to local MSP folder"
-
-  extract_orderer_tls_cert org0-orderer1
-  extract_orderer_tls_cert org0-orderer2
-  extract_orderer_tls_cert org0-orderer3
+  create_orderer_local_MSP org0 orderer1
+  create_orderer_local_MSP org0 orderer2
+  create_orderer_local_MSP org0 orderer3
+
+  create_peer_local_MSP org1 peer1
+  create_peer_local_MSP org1 peer2
+
+  create_peer_local_MSP org2 peer1
+  create_peer_local_MSP org2 peer2

  pop_fn
 }
+#
+## TLS certificates are isused by the CA's Issuer, stored in a Kube secret, and mounted into the pod at /var/hyperledger/fabric/config/tls.
+## For consistency with the Fabric-CA guide, his function copies the orderer's TLS certs into the traditional Fabric MSP / folder structure.
+#function extract_orderer_tls_cert() {
+#  local orderer=$1
+#
+#  echo 'set -x
+#
+#  mkdir -p /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/'${orderer}'.org0.example.com/tls/signcerts/
+#
+#  cp \
+#    var/hyperledger/fabric/config/tls/tls.crt \
+#    /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/'${orderer}'.org0.example.com/tls/signcerts/cert.pem
+#
+#  ' | exec kubectl -n $NS exec deploy/${orderer} -i -c main -- /bin/sh
+#}
+#
+#function extract_orderer_tls_certs() {
+#  push_fn "Extracting orderer TLS certs to local MSP folder"
+#
+#  extract_orderer_tls_cert org0-orderer1
+#  extract_orderer_tls_cert org0-orderer2
+#  extract_orderer_tls_cert org0-orderer3
+#
+#  pop_fn
+#}

 function network_up() {

@ -213,17 +177,13 @@ function network_up() {
  launch_orderers
  launch_peers

-  extract_orderer_tls_certs
+#  extract_orderer_tls_certs
 }

 function stop_services() {
  push_fn "Stopping Fabric services"

-  # These pods are busy executing `sleep MAX_INT` and do not shut down very quickly...
-#  kubectl -n $NS delete deployment/org0-admin-cli --grace-period=0 --force
-#  kubectl -n $NS delete deployment/org1-admin-cli --grace-period=0 --force
-#  kubectl -n $NS delete deployment/org2-admin-cli --grace-period=0 --force
-
+  kubectl -n $NS delete ingress --all
  kubectl -n $NS delete deployment --all
  kubectl -n $NS delete pod --all
  kubectl -n $NS delete service --all
@ -252,4 +212,6 @@ function scrub_org_volumes() {
 function network_down() {
  stop_services
  scrub_org_volumes
+
+  rm -rf $PWD/build
 }
--- a/test-network-k8s/scripts/utils.sh
+++ b/test-network-k8s/scripts/utils.sh
@ -25,6 +25,8 @@ function logging_init() {
 function exit_fn() {
  rc=$?

+  set +x
+
  # Write an error icon to the current logging statement.
  if [ "0" -ne $rc ]; then
    pop_fn $rc
@ -73,3 +75,12 @@ function pop_fn() {
  echo "" >> ${LOG_FILE}
 }

+# Apply the current environment to a k8s template and apply to the cluster.
+function apply_template() {
+
+  echo "Applying template $1:"
+  cat $1 | envsubst
+
+  cat $1 | envsubst | kubectl -n $NS apply -f -
+}
+