Access the test network services via a local Nginx ingress controller.

Signed-off-by: Josh Kneubuhl <jkneubuh@us.ibm.com>

test-network-k8s/.gitignore

@@ -3,3 +3,4 @@ network.log
 network-debug.log 
 build/
 .env
+bin/

test-network-k8s/config/org0/configtx.yaml

@@ -25,7 +25,7 @@ Organizations:
     ID: OrdererMSP
 
     # MSPDir is the filesystem path which contains the MSP configuration
-    MSPDir: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp
+    MSPDir: ../../build/channel-msp/ordererOrganizations/org0/msp
 
     # Policies defines the set of policies at this level of the config tree
     # For organization policies, their canonical path is usually
@@ -54,7 +54,7 @@ Organizations:
     # ID to load the MSP definition as
     ID: Org1MSP
 
-    MSPDir: /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/msp
+    MSPDir: ../../build/channel-msp/peerOrganizations/org1/msp
 
     # Policies defines the set of policies at this level of the config tree
     # For organization policies, their canonical path is usually
@@ -89,7 +89,7 @@ Organizations:
     # ID to load the MSP definition as
     ID: Org2MSP
 
-    MSPDir: /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/msp
+    MSPDir: ../../build/channel-msp/peerOrganizations/org2/msp
 
     # Policies defines the set of policies at this level of the config tree
     # For organization policies, their canonical path is usually
@@ -224,16 +224,16 @@ Orderer: &OrdererDefaults
     Consenters:
       - Host: org0-orderer1
         Port: 6050
-        ClientTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/tls/signcerts/cert.pem
+        ClientTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem
-        ServerTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/tls/signcerts/cert.pem
+        ServerTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem
       - Host: org0-orderer2
         Port: 6050
-        ClientTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/tls/signcerts/cert.pem
+        ClientTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer2/tls/signcerts/tls-cert.pem
-        ServerTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/tls/signcerts/cert.pem
+        ServerTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer2/tls/signcerts/tls-cert.pem
       - Host: org0-orderer3
         Port: 6050
-        ClientTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/tls/signcerts/cert.pem
+        ClientTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer3/tls/signcerts/tls-cert.pem
-        ServerTLSCert: /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/tls/signcerts/cert.pem
+        ServerTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer3/tls/signcerts/tls-cert.pem
 
 
     # Options to be specified for all the etcd/raft nodes. The values here

test-network-k8s/kube/org0/org0-admin-cli.yaml

@@ -1,61 +0,0 @@
-#
-# Copyright IBM Corp. All Rights Reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: org0-admin-cli
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: org0-admin-cli
-  template:
-    metadata:
-      labels:
-        app: org0-admin-cli
-    spec:
-      containers:
-        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-tools:{{FABRIC_VERSION}}
-          imagePullPolicy: IfNotPresent
-          env:
-            - name: FABRIC_CFG_PATH
-              value: /var/hyperledger/fabric/config
-          args:
-            - sleep
-            - "2147483647"
-          workingDir: /root
-          volumeMounts:
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-            - name: fabric-config
-              mountPath: /var/hyperledger/fabric/config
-
-      # This init container will unfurl all of the MSP archives listed in the msp-config config map.
-      initContainers:
-        - name: msp-unfurl
-          image: busybox
-          command:
-            - sh
-            - -c
-            - "for msp in $(ls /msp/msp-*.tgz); do echo $msp && tar zxvf $msp -C /var/hyperledger/fabric; done"
-          volumeMounts:
-            - name: msp-config
-              mountPath: /msp
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-
-      volumes:
-        - name: fabric-volume
-          persistentVolumeClaim:
-            claimName: fabric-org0
-        - name: fabric-config
-          configMap:
-            name: org0-config
-        - name: msp-config
-          configMap:
-            name: msp-config

test-network-k8s/kube/org0/org0-ca.yaml

@@ -17,6 +17,7 @@ spec:
     - localhost
     - org0-ca
     - org0-ca.test-network.svc.cluster.local
+    - org0-ca.${DOMAIN}
   ipAddresses:
     - 127.0.0.1
   secretName: org0-ca-tls-cert
@@ -40,7 +41,7 @@ spec:
     spec:
       containers:
         - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-ca:{{FABRIC_CA_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-ca:${FABRIC_CA_VERSION}
           imagePullPolicy: IfNotPresent
           env:
             - name: FABRIC_CA_SERVER_CA_NAME
@@ -89,8 +90,35 @@ metadata:
   name: org0-ca
 spec:
   ports:
-    - name: tls
+    - name: https
       port: 443
       protocol: TCP
   selector:
-    app: org0-ca
+    app: org0-ca
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org0-ca
+  name: org0-ca
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org0-ca.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-ca
+                port:
+                  name: https
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+      - org0-ca.${DOMAIN}

test-network-k8s/kube/org0/org0-orderer1.yaml

@@ -9,7 +9,7 @@ apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: org0-orderer1-tls-cert
-  namespace: test-network
+  namespace: ${NS}
 spec:
   isCA: false
   privateKey:
@@ -18,7 +18,9 @@ spec:
   dnsNames:
     - localhost
     - org0-orderer1
-    - org0-orderer1.test-network.svc.cluster.local
+    - org0-orderer1.${NS}.svc.cluster.local
+    - org0-orderer1.${DOMAIN}
+    - org0-orderer1-admin.${DOMAIN}
   ipAddresses:
     - 127.0.0.1
   secretName: org0-orderer1-tls-cert
@@ -42,6 +44,12 @@ data:
   ORDERER_GENERAL_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
   ORDERER_GENERAL_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
   ORDERER_GENERAL_BOOTSTRAPMETHOD: none
+  ORDERER_ADMIN_TLS_ENABLED: "true"
+  ORDERER_ADMIN_TLS_CERTIFICATE: /var/hyperledger/fabric/config/tls/tls.crt
+  ORDERER_ADMIN_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
+  ORDERER_ADMIN_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
+  # Authenticate client connections with the org's ecert / admin user enrollments
+  ORDERER_ADMIN_TLS_CLIENTROOTCAS: "[/var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp/cacerts/org0-ca.pem]"
   ORDERER_FILELEDGER_LOCATION: /var/hyperledger/fabric/data/orderer1
   ORDERER_CONSENSUS_WALDIR: /var/hyperledger/fabric/data/orderer1/etcdraft/wal
   ORDERER_CONSENSUS_SNAPDIR: /var/hyperledger/fabric/data/orderer1/etcdraft/wal
@@ -65,7 +73,7 @@ spec:
     spec:
       containers:
         - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-orderer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-orderer:${FABRIC_VERSION}
           imagePullPolicy: IfNotPresent
           envFrom:
             - configMapRef:
@@ -109,4 +117,43 @@ spec:
       port: 9443
       protocol: TCP
   selector:
-    app: org0-orderer1
+    app: org0-orderer1
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org0-orderer1
+  name: org0-orderer1
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org0-orderer1.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer1
+                port:
+                  name: general
+            path: /
+            pathType: ImplementationSpecific
+    - host: org0-orderer1-admin.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer1
+                port:
+                  name: admin
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org0-orderer1.${DOMAIN}
+    - hosts:
+        - org0-orderer1-admin.${DOMAIN}

test-network-k8s/kube/org0/org0-orderer2.yaml

@@ -3,12 +3,13 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 #
+
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: org0-orderer2-tls-cert
-  namespace: test-network
+  namespace: ${NS}
 spec:
   isCA: false
   privateKey:
@@ -17,7 +18,9 @@ spec:
   dnsNames:
     - localhost
     - org0-orderer2
-    - org0-orderer2.test-network.svc.cluster.local
+    - org0-orderer2.${NS}.svc.cluster.local
+    - org0-orderer2.${DOMAIN}
+    - org0-orderer2-admin.${DOMAIN}
   ipAddresses:
     - 127.0.0.1
   secretName: org0-orderer2-tls-cert
@@ -41,6 +44,12 @@ data:
   ORDERER_GENERAL_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
   ORDERER_GENERAL_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
   ORDERER_GENERAL_BOOTSTRAPMETHOD: none
+  ORDERER_ADMIN_TLS_ENABLED: "true"
+  ORDERER_ADMIN_TLS_CERTIFICATE: /var/hyperledger/fabric/config/tls/tls.crt
+  ORDERER_ADMIN_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
+  ORDERER_ADMIN_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
+  # Authenticate client connections with the org's ecert / admin user enrollments
+  ORDERER_ADMIN_TLS_CLIENTROOTCAS: "[/var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/msp/cacerts/org0-ca.pem]"
   ORDERER_FILELEDGER_LOCATION: /var/hyperledger/fabric/data/orderer2
   ORDERER_CONSENSUS_WALDIR: /var/hyperledger/fabric/data/orderer2/etcdraft/wal
   ORDERER_CONSENSUS_SNAPDIR: /var/hyperledger/fabric/data/orderer2/etcdraft/wal
@@ -64,7 +73,7 @@ spec:
     spec:
       containers:
         - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-orderer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-orderer:${FABRIC_VERSION}
           imagePullPolicy: IfNotPresent
           envFrom:
             - configMapRef:
@@ -108,4 +117,43 @@ spec:
       port: 9443
       protocol: TCP
   selector:
-    app: org0-orderer2
+    app: org0-orderer2
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org0-orderer2
+  name: org0-orderer2
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org0-orderer2.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer2
+                port:
+                  name: general
+            path: /
+            pathType: ImplementationSpecific
+    - host: org0-orderer2-admin.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer2
+                port:
+                  name: admin
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org0-orderer2.${DOMAIN}
+    - hosts:
+        - org0-orderer2-admin.${DOMAIN}

test-network-k8s/kube/org0/org0-orderer3.yaml

@@ -3,12 +3,13 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 #
+
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: org0-orderer3-tls-cert
-  namespace: test-network
+  namespace: ${NS}
 spec:
   isCA: false
   privateKey:
@@ -17,13 +18,15 @@ spec:
   dnsNames:
     - localhost
     - org0-orderer3
-    - org0-orderer3.test-network.svc.cluster.local
+    - org0-orderer3.${NS}.svc.cluster.local
+    - org0-orderer3.${DOMAIN}
+    - org0-orderer3-admin.${DOMAIN}
   ipAddresses:
     - 127.0.0.1
   secretName: org0-orderer3-tls-cert
   issuerRef:
     name: org0-tls-cert-issuer
-    
+
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -41,6 +44,12 @@ data:
   ORDERER_GENERAL_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
   ORDERER_GENERAL_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
   ORDERER_GENERAL_BOOTSTRAPMETHOD: none
+  ORDERER_ADMIN_TLS_ENABLED: "true"
+  ORDERER_ADMIN_TLS_CERTIFICATE: /var/hyperledger/fabric/config/tls/tls.crt
+  ORDERER_ADMIN_TLS_ROOTCAS: /var/hyperledger/fabric/config/tls/ca.crt
+  ORDERER_ADMIN_TLS_PRIVATEKEY: /var/hyperledger/fabric/config/tls/tls.key
+  # Authenticate client connections with the org's ecert / admin user enrollments
+  ORDERER_ADMIN_TLS_CLIENTROOTCAS: "[/var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/msp/cacerts/org0-ca.pem]"
   ORDERER_FILELEDGER_LOCATION: /var/hyperledger/fabric/data/orderer3
   ORDERER_CONSENSUS_WALDIR: /var/hyperledger/fabric/data/orderer3/etcdraft/wal
   ORDERER_CONSENSUS_SNAPDIR: /var/hyperledger/fabric/data/orderer3/etcdraft/wal
@@ -64,7 +73,7 @@ spec:
     spec:
       containers:
         - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-orderer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-orderer:${FABRIC_VERSION}
           imagePullPolicy: IfNotPresent
           envFrom:
             - configMapRef:
@@ -108,4 +117,43 @@ spec:
       port: 9443
       protocol: TCP
   selector:
-    app: org0-orderer3
+    app: org0-orderer3
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org0-orderer3
+  name: org0-orderer3
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org0-orderer3.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer3
+                port:
+                  name: general
+            path: /
+            pathType: ImplementationSpecific
+    - host: org0-orderer3-admin.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org0-orderer3
+                port:
+                  name: admin
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org0-orderer3.${DOMAIN}
+    - hosts:
+        - org0-orderer3-admin.${DOMAIN}

test-network-k8s/kube/org1/org1-admin-cli.yaml

@@ -1,65 +0,0 @@
-#
-# Copyright IBM Corp. All Rights Reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: org1-admin-cli
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: org1-admin-cli
-  template:
-    metadata:
-      labels:
-        app: org1-admin-cli
-    spec:
-      containers:
-        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-tools:{{FABRIC_VERSION}}
-          imagePullPolicy: IfNotPresent
-          env:
-            - name: FABRIC_CFG_PATH
-              value: /var/hyperledger/fabric/config
-            - name: CORE_PEER_MSPCONFIGPATH
-              value: /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp
-            - name: CORE_PEER_TLS_ROOTCERT_FILE
-              value: /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/msp/tlscacerts/org1-tls-ca.pem
-          args:
-            - sleep
-            - "2147483647"
-          workingDir: /root
-          volumeMounts:
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-            - name: fabric-config
-              mountPath: /var/hyperledger/fabric/config
-
-      # This init container will unfurl all of the MSP archives listed in the msp-config config map.
-      initContainers:
-        - name: msp-unfurl
-          image: busybox
-          command:
-            - sh
-            - -c
-            - "for msp in $(ls /msp/msp-*.tgz); do echo $msp && tar zxvf $msp -C /var/hyperledger/fabric; done"
-          volumeMounts:
-            - name: msp-config
-              mountPath: /msp
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-
-      volumes:
-        - name: fabric-volume
-          persistentVolumeClaim:
-            claimName: fabric-org1
-        - name: fabric-config
-          configMap:
-            name: org1-config
-        - name: msp-config
-          configMap:
-            name: msp-config

test-network-k8s/kube/org1/org1-ca.yaml

@@ -17,6 +17,7 @@ spec:
     - localhost
     - org1-ca
     - org1-ca.test-network.svc.cluster.local
+    - org1-ca.${DOMAIN}
   ipAddresses:
     - 127.0.0.1
   secretName: org1-ca-tls-cert
@@ -40,7 +41,7 @@ spec:
     spec:
       containers:
         - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-ca:{{FABRIC_CA_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-ca:${FABRIC_CA_VERSION}
           imagePullPolicy: IfNotPresent
           env:
             - name: FABRIC_CA_SERVER_CA_NAME
@@ -88,8 +89,36 @@ metadata:
   name: org1-ca
 spec:
   ports:
-    - name: tls
+    - name: https
       port: 443
       protocol: TCP
   selector:
-    app: org1-ca
+    app: org1-ca
+    
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org1-ca
+  name: org1-ca
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org1-ca.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org1-ca
+                port:
+                  name: https
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org1-ca.${DOMAIN}
+  

test-network-k8s/kube/org1/org1-peer1.yaml

@@ -17,8 +17,10 @@ spec:
   dnsNames:
     - localhost
     - org1-peer1
-    - org1-peer-gateway-svc
     - org1-peer1.test-network.svc.cluster.local
+    - org1-peer1.${DOMAIN}
+    - org1-peer-gateway-svc
+    - org1-peer-gateway-svc.${DOMAIN}
   ipAddresses:
     - 127.0.0.1
   secretName: org1-peer1-tls-cert
@@ -74,7 +76,7 @@ spec:
     spec:
       containers:
         - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-peer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-peer:${FABRIC_VERSION}
           imagePullPolicy: IfNotPresent
           envFrom:
             - configMapRef:
@@ -118,7 +120,7 @@ metadata:
   name: org1-peer1
 spec:
   ports:
-    - name: gossip
+    - name: grpc
       port: 7051
       protocol: TCP
     - name: chaincode
@@ -136,8 +138,46 @@ metadata:
   name: org1-peer-gateway-svc
 spec:
   ports:
-    - name: gossip
+    - name: grpc
       port: 7051
       protocol: TCP
   selector:
-    org: org1
+    org: org1
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org1-peer1
+  name: org1-peer1
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org1-peer1.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org1-peer1
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+    - host: org1-peer-gateway-svc.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org1-peer1
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org1-peer1.${DOMAIN}
+    - hosts:
+        - org1-peer-gateway-svc.${DOMAIN}

test-network-k8s/kube/org1/org1-peer2.yaml

@@ -19,6 +19,7 @@ spec:
     - org1-peer2
     - org1-peer-gateway-svc
     - org1-peer2.test-network.svc.cluster.local
+    - org1-peer2.${DOMAIN}
   ipAddresses:
     - 127.0.0.1
   secretName: org1-peer2-tls-cert
@@ -75,7 +76,7 @@ spec:
     spec:
       containers:
         - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-peer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-peer:${FABRIC_VERSION}
           imagePullPolicy: IfNotPresent
           envFrom:
             - configMapRef:
@@ -119,7 +120,7 @@ metadata:
   name: org1-peer2
 spec:
   ports:
-    - name: gossip
+    - name: grpc
       port: 7051
       protocol: TCP
     - name: chaincode
@@ -129,4 +130,30 @@ spec:
       port: 9443
       protocol: TCP
   selector:
-    app: org1-peer2
+    app: org1-peer2
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org1-peer2
+  name: org1-peer2
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org1-peer2.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org1-peer2
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org1-peer2.${DOMAIN}

test-network-k8s/kube/org2/org2-admin-cli.yaml

@@ -1,65 +0,0 @@
-#
-# Copyright IBM Corp. All Rights Reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: org2-admin-cli
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: org2-admin-cli
-  template:
-    metadata:
-      labels:
-        app: org2-admin-cli
-    spec:
-      containers:
-        - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-tools:{{FABRIC_VERSION}}
-          imagePullPolicy: IfNotPresent
-          env:
-            - name: FABRIC_CFG_PATH
-              value: /var/hyperledger/fabric/config
-            - name: CORE_PEER_MSPCONFIGPATH
-              value: /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp
-            - name: CORE_PEER_TLS_ROOTCERT_FILE
-              value: /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/msp/tlscacerts/org2-tls-ca.pem
-          args:
-            - sleep
-            - "2147483647"
-          workingDir: /root
-          volumeMounts:
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-            - name: fabric-config
-              mountPath: /var/hyperledger/fabric/config
-
-      # This init container will unfurl all of the MSP archives listed in the msp-config config map.
-      initContainers:
-        - name: msp-unfurl
-          image: busybox
-          command:
-            - sh
-            - -c
-            - "for msp in $(ls /msp/msp-*.tgz); do echo $msp && tar zxvf $msp -C /var/hyperledger/fabric; done"
-          volumeMounts:
-            - name: msp-config
-              mountPath: /msp
-            - name: fabric-volume
-              mountPath: /var/hyperledger
-
-      volumes:
-        - name: fabric-volume
-          persistentVolumeClaim:
-            claimName: fabric-org2
-        - name: fabric-config
-          configMap:
-            name: org2-config
-        - name: msp-config
-          configMap:
-            name: msp-config

test-network-k8s/kube/org2/org2-ca.yaml

@@ -17,6 +17,7 @@ spec:
     - localhost
     - org2-ca
     - org2-ca.test-network.svc.cluster.local
+    - org2-ca.${DOMAIN}
   ipAddresses:
     - 127.0.0.1
   secretName: org2-ca-tls-cert
@@ -40,7 +41,7 @@ spec:
     spec:
       containers:
         - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-ca:{{FABRIC_CA_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-ca:${FABRIC_CA_VERSION}
           imagePullPolicy: IfNotPresent
           env:
             - name: FABRIC_CA_SERVER_CA_NAME
@@ -88,8 +89,36 @@ metadata:
   name: org2-ca
 spec:
   ports:
-    - name: tls
+    - name: https
       port: 443
       protocol: TCP
   selector:
-    app: org2-ca
+    app: org2-ca
+    
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org2-ca
+  name: org2-ca
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org2-ca.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org2-ca
+                port:
+                  name: https
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org2-ca.${DOMAIN}
+    

test-network-k8s/kube/org2/org2-peer1.yaml

@@ -17,8 +17,10 @@ spec:
   dnsNames:
     - localhost
     - org2-peer1
-    - org2-peer-gateway-svc
     - org2-peer1.test-network.svc.cluster.local
+    - org2-peer1.${DOMAIN}
+    - org2-peer-gateway-svc
+    - org2-peer-gateway-svc.${DOMAIN}
   ipAddresses:
     - 127.0.0.1
   secretName: org2-peer1-tls-cert
@@ -74,7 +76,7 @@ spec:
     spec:
       containers:
         - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-peer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-peer:${FABRIC_VERSION}
           imagePullPolicy: IfNotPresent
           envFrom:
             - configMapRef:
@@ -118,7 +120,7 @@ metadata:
   name: org2-peer1
 spec:
   ports:
-    - name: gossip
+    - name: grpc
       port: 7051
       protocol: TCP
     - name: chaincode
@@ -136,8 +138,46 @@ metadata:
   name: org2-peer-gateway-svc
 spec:
   ports:
-    - name: gossip
+    - name: grpc
       port: 7051
       protocol: TCP
   selector:
     org: org2
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org2-peer1
+  name: org2-peer1
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org2-peer1.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org2-peer1
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+    - host: org2-peer-gateway-svc.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org2-peer1
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org2-peer1.${DOMAIN}
+    - hosts:
+        - org2-peer-gateway-svc.${DOMAIN}

test-network-k8s/kube/org2/org2-peer2.yaml

@@ -19,6 +19,7 @@ spec:
     - org2-peer2
     - org2-peer-gateway-svc
     - org2-peer2.test-network.svc.cluster.local
+    - org2-peer2.${DOMAIN}
   ipAddresses:
     - 127.0.0.1
   secretName: org2-peer2-tls-cert
@@ -74,7 +75,7 @@ spec:
     spec:
       containers:
         - name: main
-          image: {{FABRIC_CONTAINER_REGISTRY}}/fabric-peer:{{FABRIC_VERSION}}
+          image: ${FABRIC_CONTAINER_REGISTRY}/fabric-peer:${FABRIC_VERSION}
           imagePullPolicy: IfNotPresent
           envFrom:
             - configMapRef:
@@ -118,7 +119,7 @@ metadata:
   name: org2-peer2
 spec:
   ports:
-    - name: gossip
+    - name: grpc
       port: 7051
       protocol: TCP
     - name: chaincode
@@ -129,3 +130,29 @@ spec:
       protocol: TCP
   selector:
     app: org2-peer2
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: 60s
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  labels:
+    app: org2-peer2
+  name: org2-peer2
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: org2-peer2.${DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: org2-peer2
+                port:
+                  name: grpc
+            path: /
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - org2-peer2.${DOMAIN}

test-network-k8s/network

@@ -20,14 +20,17 @@ set -o errexit
 # todo: track down a nasty bug whereby the CA service endpoints (kube services) will occasionally reject TCP connections after network down/up.  This is patched by introducing a 10s sleep after the deployments are up...
 # todo: refactor query/invoke to specify chaincode name (-n param)
 
-CONTAINER_CLI=${CONTAINER_CLI:-docker}
+export CONTAINER_CLI=${CONTAINER_CLI:-docker}
-FABRIC_VERSION=${TEST_NETWORK_FABRIC_VERSION:-2.4}
+export FABRIC_VERSION=${TEST_NETWORK_FABRIC_VERSION:-2.4.3}
-FABRIC_CA_VERSION=${TEST_NETWORK_FABRIC_CA_VERSION:-1.5}
+export FABRIC_CA_VERSION=${TEST_NETWORK_FABRIC_CA_VERSION:-1.5.2}
-FABRIC_CONTAINER_REGISTRY=${TEST_NETWORK_FABRIC_CONTAINER_REGISTRY:-hyperledger}
+export FABRIC_CONTAINER_REGISTRY=${TEST_NETWORK_FABRIC_CONTAINER_REGISTRY:-hyperledger}
-NETWORK_NAME=${TEST_NETWORK_NAME:-test-network}
+export NETWORK_NAME=${TEST_NETWORK_NAME:-test-network}
-CLUSTER_NAME=${TEST_NETWORK_KIND_CLUSTER_NAME:-kind}
+export CLUSTER_NAME=${TEST_NETWORK_KIND_CLUSTER_NAME:-kind}
-NS=${TEST_NETWORK_KUBE_NAMESPACE:-${NETWORK_NAME}}
+export NS=${TEST_NETWORK_KUBE_NAMESPACE:-${NETWORK_NAME}}
-CHANNEL_NAME=${TEST_NETWORK_CHANNEL_NAME:-mychannel}
+export DOMAIN=${TEST_NETWORK_DOMAIN:-vcap.me}
+export CHANNEL_NAME=${TEST_NETWORK_CHANNEL_NAME:-mychannel}
+export TEMP_DIR=${PWD}/build
+
 LOG_FILE=${TEST_NETWORK_LOG_FILE:-network.log}
 DEBUG_FILE=${TEST_NETWORK_DEBUG_FILE:-network-debug.log}
 LOCAL_REGISTRY_NAME=${TEST_NETWORK_LOCAL_REGISTRY_NAME:-kind-registry}
@@ -50,6 +53,7 @@ function print_help() {
   log "Fabric CA Version    \t: ${FABRIC_CA_VERSION}"
   log "Container Registry   \t: ${FABRIC_CONTAINER_REGISTRY}"
   log "Network name       \t\t: ${NETWORK_NAME}"
+  log "Ingress domain     \t\t: ${DOMAIN}"
   log "Channel name       \t\t: ${CHANNEL_NAME}"
   log
   log "--- Chaincode Information"

test-network-k8s/scripts/channel.sh

@@ -5,197 +5,272 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-function create_channel_org_MSP() {
+function channel_up() {
+  set -x
+
+  register_org_admins
+  enroll_org_admins
+
+  create_channel_MSP
+  create_genesis_block
+
+  join_channel_orderers
+  join_channel_peers
+}
+
+function register_org_admins() {
+  push_fn "Registering org Admin users"
+
+  register_org_admin org0 org0admin org0adminpw
+  register_org_admin org1 org1admin org1adminpw
+  register_org_admin org2 org2admin org2adminpw
+
+  pop_fn
+}
+
+# Register the org admin user
+function register_org_admin() {
+  local type=admin
   local org=$1
-  local org_type=$2
+  local id_name=$2
-  local ecert_ca=${org}-ca
+  local id_secret=$3
-  
+  local ca_name=${org}-ca
-  echo 'set -x
- 
-  mkdir -p /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/cacerts
-  cp \
-    $FABRIC_CA_CLIENT_HOME/'${ecert_ca}'/rcaadmin/msp/cacerts/'${ecert_ca}'.pem \
-    /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/cacerts
 
-  mkdir -p /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/tlscacerts
+  echo "Registering org admin $username"
-  cp \
-    /var/hyperledger/fabric/config/tls/ca.crt \
-    /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/tlscacerts/'${org}'-tls-ca.pem
 
-  echo "NodeOUs:
+  cat <<EOF | kubectl -n $NS exec deploy/${ca_name} -i -- /bin/sh
-    Enable: true
+
-    ClientOUIdentifier:
+  set -x
-      Certificate: cacerts/'${ecert_ca}'.pem
+  export FABRIC_CA_CLIENT_HOME=/var/hyperledger/fabric-ca-client
-      OrganizationalUnitIdentifier: client
+  export FABRIC_CA_CLIENT_TLS_CERTFILES=/var/hyperledger/fabric/config/tls/ca.crt
-    PeerOUIdentifier:
+
-      Certificate: cacerts/'${ecert_ca}'.pem
+  fabric-ca-client register \
-      OrganizationalUnitIdentifier: peer
+    --id.name ${id_name} \
-    AdminOUIdentifier:
+    --id.secret ${id_secret} \
-      Certificate: cacerts/'${ecert_ca}'.pem
+    --id.type ${type} \
-      OrganizationalUnitIdentifier: admin
+    --url https://${ca_name} \
-    OrdererOUIdentifier:
+    --mspdir /var/hyperledger/fabric-ca-client/${ca_name}/rcaadmin/msp \
-      Certificate: cacerts/'${ecert_ca}'.pem
+    --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
-      OrganizationalUnitIdentifier: orderer "> /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/config.yaml
+EOF
-      
+}
-  ' | exec kubectl -n $NS exec deploy/${ecert_ca} -i -- /bin/sh
+
+function enroll_org_admins() {
+  push_fn "Enrolling org Admin users"
+
+  enroll_org_admin orderer  org0 org0admin org0adminpw
+  enroll_org_admin peer     org1 org1admin org1adminpw
+  enroll_org_admin peer     org2 org2admin org2adminpw
+
+  pop_fn
+}
+
+# Enroll the admin client to the local certificate storage folder.
+function enroll_org_admin() {
+  local type=$1
+  local org=$2
+  local username=$3
+  local password=$4
+
+  echo "Enrolling $type org admin $username"
+
+  ENROLLMENTS_DIR=${TEMP_DIR}/enrollments
+  ORG_ADMIN_DIR=${ENROLLMENTS_DIR}/${org}/users/${username}
+
+  # skip the enrollment if the admin certificate is available.
+  if [ -f "${ORG_ADMIN_DIR}/msp/keystore/key.pem" ]; then
+    echo "Found an existing admin enrollment at ${ORG_ADMIN_DIR}"
+    return
+  fi
+
+  # Determine the CA information and TLS certificate
+  CA_NAME=${org}-ca
+  CA_DIR=${TEMP_DIR}/cas/${CA_NAME}
+  mkdir -p ${CA_DIR}
+
+  CA_AUTH=${username}:${password}
+  CA_HOST=${CA_NAME}.${DOMAIN}
+  CA_PORT=443
+  CA_URL=https://${CA_AUTH}@${CA_HOST}:${CA_PORT}
+
+  # Read the CA's TLS certificate from the cert-manager CA secret
+  echo "retrieving ${org}-ca TLS cert"
+  kubectl -n $NS get secret ${org}-ca-tls-cert -o json \
+    | jq -r .data.\"tls.crt\" \
+    | base64 -d \
+    > ${CA_DIR}/tls-cert.pem
+
+  # enroll the org admin
+  FABRIC_CA_CLIENT_HOME=${ORG_ADMIN_DIR} fabric-ca-client enroll \
+    --url ${CA_URL} \
+    --tls.certfiles ${CA_DIR}/tls-cert.pem
+
+
+  # Construct an msp config.yaml
+  CA_CERT_NAME=${CA_NAME}-$(echo $DOMAIN | tr -s . -)-${CA_PORT}.pem
+
+  create_msp_config_yaml ${CA_NAME} ${CA_CERT_NAME} ${ORG_ADMIN_DIR}/msp
+
+  # private keys are hashed by name, but we only support one enrollment.
+  # test-network examples refer to this as "server.key", which is incorrect.
+  # This is the private key used to endorse transactions using the admin's
+  # public key.
+  mv ${ORG_ADMIN_DIR}/msp/keystore/*_sk ${ORG_ADMIN_DIR}/msp/keystore/key.pem
+}
+
+# create an enrollment MSP config.yaml
+function create_msp_config_yaml() {
+  local ca_name=$1
+  local ca_cert_name=$2
+  local msp_dir=$3
+  echo "Creating msp config ${msp_dir}/config.yaml with cert ${ca_cert_name}"
+
+  cat << EOF > ${msp_dir}/config.yaml
+NodeOUs:
+  Enable: true
+  ClientOUIdentifier:
+    Certificate: cacerts/${ca_cert_name}
+    OrganizationalUnitIdentifier: client
+  PeerOUIdentifier:
+    Certificate: cacerts/${ca_cert_name}
+    OrganizationalUnitIdentifier: peer
+  AdminOUIdentifier:
+    Certificate: cacerts/${ca_cert_name}
+    OrganizationalUnitIdentifier: admin
+  OrdererOUIdentifier:
+    Certificate: cacerts/${ca_cert_name}
+    OrganizationalUnitIdentifier: orderer
+EOF
 }
 
 function create_channel_MSP() {
   push_fn "Creating channel MSP"
 
-  create_channel_org_MSP org0 orderer 
+  create_channel_org_MSP org0 orderer
   create_channel_org_MSP org1 peer
   create_channel_org_MSP org2 peer
 
-  pop_fn
+  extract_orderer_tls_cert org0 orderer1
-}
+  extract_orderer_tls_cert org0 orderer2
-
+  extract_orderer_tls_cert org0 orderer3
-function aggregate_channel_MSP() {
-  push_fn "Aggregating channel MSP"
-
-  rm -rf ./build/msp/
-  mkdir -p ./build/msp
-  
-  kubectl -n $NS exec deploy/org0-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/ordererOrganizations/org0.example.com/msp > build/msp/msp-org0.example.com.tgz
-  kubectl -n $NS exec deploy/org1-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/peerOrganizations/org1.example.com/msp > build/msp/msp-org1.example.com.tgz
-  kubectl -n $NS exec deploy/org2-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/peerOrganizations/org2.example.com/msp > build/msp/msp-org2.example.com.tgz
-
-  kubectl -n $NS delete configmap msp-config || true
-  kubectl -n $NS create configmap msp-config --from-file=build/msp/
 
   pop_fn
 }
 
-function launch_admin_CLIs() {
+function create_channel_org_MSP() {
-  push_fn "Launching admin CLIs"
+  local org=$1
+  local type=$2
+  local ca_name=${org}-ca
 
-  launch kube/org0/org0-admin-cli.yaml
+  ORG_MSP_DIR=${TEMP_DIR}/channel-msp/${type}Organizations/${org}/msp
-  launch kube/org1/org1-admin-cli.yaml
+  mkdir -p ${ORG_MSP_DIR}/cacerts
-  launch kube/org2/org2-admin-cli.yaml
+  mkdir -p ${ORG_MSP_DIR}/tlscacerts
 
-  kubectl -n $NS rollout status deploy/org0-admin-cli
+  # extract the CA's signing authority from the CA/cainfo response
-  kubectl -n $NS rollout status deploy/org1-admin-cli
+  curl -s \
-  kubectl -n $NS rollout status deploy/org2-admin-cli
+    --cacert ${TEMP_DIR}/cas/${ca_name}/tls-cert.pem \
+    https://${ca_name}.${DOMAIN}/cainfo \
+    | jq -r .result.CAChain \
+    | base64 -d \
+    > ${ORG_MSP_DIR}/cacerts/ca-signcert.pem
 
-  pop_fn
+  # extract the CA's TLS CA certificate from the cert-manager secret
+  kubectl -n $NS get secret ${ca_name}-tls-cert -o json \
+    | jq -r .data.\"ca.crt\" \
+    | base64 -d \
+    > ${ORG_MSP_DIR}/tlscacerts/tlsca-signcert.pem
+
+  # create an MSP config.yaml with the CA's signing certificate
+  create_msp_config_yaml ${ca_name} ca-signcert.pem ${ORG_MSP_DIR}
+}
+
+# Extract an orderer's TLS signing certificate for inclusion in the channel config block
+function extract_orderer_tls_cert() {
+  local org=$1
+  local orderer=$2
+
+  echo "Extracting TLS cert for $org $orderer"
+
+  ORDERER_TLS_DIR=${TEMP_DIR}/channel-msp/ordererOrganizations/${org}/orderers/${org}-${orderer}/tls
+  mkdir -p $ORDERER_TLS_DIR/signcerts
+
+  kubectl -n $NS get secret ${org}-${orderer}-tls-cert -o json \
+    | jq -r .data.\"tls.crt\" \
+    | base64 -d \
+    > ${ORDERER_TLS_DIR}/signcerts/tls-cert.pem
 }
 
 function create_genesis_block() {
-  push_fn "Creating channel \"${CHANNEL_NAME}\""
+  push_fn "Creating channel genesis block"
 
-  echo 'set -x
+  FABRIC_CFG_PATH=${PWD}/config/org0 \
-  configtxgen -profile TwoOrgsApplicationGenesis -channelID '${CHANNEL_NAME}' -outputBlock genesis_block.pb
+    configtxgen \
-  # configtxgen -inspectBlock genesis_block.pb
+      -profile TwoOrgsApplicationGenesis \
-  
+      -channelID $CHANNEL_NAME \
-  osnadmin channel join --orderer-address org0-orderer1:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb
+      -outputBlock ${TEMP_DIR}/genesis_block.pb
-  osnadmin channel join --orderer-address org0-orderer2:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb
+
-  osnadmin channel join --orderer-address org0-orderer3:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb
+  # configtxgen -inspectBlock ${TEMP_DIR}/genesis_block.pb
-  
+
-  ' | exec kubectl -n $NS exec deploy/org0-admin-cli -i -- /bin/bash
+  pop_fn
-  
+}
-  # todo: readiness / liveiness equivalent for channel ?    Needs a little bit to settle before peers can join. 
+
+function join_channel_orderers() {
+  push_fn "Joining orderers to channel ${CHANNEL_NAME}"
+
+  join_channel_orderer org0 orderer1
+  join_channel_orderer org0 orderer2
+  join_channel_orderer org0 orderer3
+
+  # todo: readiness / liveiness equivalent for channel?  Needs a little bit to settle before peers can join.
   sleep 10
 
   pop_fn
 }
 
-function join_org_peers() {
+# Request from the channel ADMIN api that the orderer joins the target channel
+function join_channel_orderer() {
   local org=$1
-  push_fn "Joining ${org} peers to channel \"${CHANNEL_NAME}\""
+  local orderer=$2
 
-  echo 'set -x
+  # The client certificate presented in this case is the admin user's enrollment key.  This is a stronger assertion
-  # Fetch the genesis block from an orderer
+  # of identity than the Docker Compose network, which transmits the orderer node's TLS key pair directly
-  peer channel \
+  osnadmin channel join \
-    fetch oldest \
+    --orderer-address ${org}-${orderer}-admin.${DOMAIN} \
-    genesis_block.pb \
+    --ca-file ${TEMP_DIR}/channel-msp/ordererOrganizations/${org}/orderers/${org}-${orderer}/tls/signcerts/tls-cert.pem \
-    -c '${CHANNEL_NAME}' \
+    --client-cert ${TEMP_DIR}/enrollments/${org}/users/${org}admin/msp/signcerts/cert.pem \
-    -o org0-orderer1:6050 \
+    --client-key ${TEMP_DIR}/enrollments/${org}/users/${org}admin/msp/keystore/key.pem \
-    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
+    --channelID ${CHANNEL_NAME} \
-
+    --config-block ${TEMP_DIR}/genesis_block.pb
-  # Join peer1 to the channel.
-  CORE_PEER_ADDRESS='${org}'-peer1:7051 \
-  peer channel \
-    join \
-    -b genesis_block.pb \
-    -o org0-orderer1:6050 \
-    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
-
-  # Join peer2 to the channel.
-  CORE_PEER_ADDRESS='${org}'-peer2:7051 \
-  peer channel \
-    join \
-    -b genesis_block.pb \
-    -o org0-orderer1:6050 \
-    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
-
-  ' | exec kubectl -n $NS exec deploy/${org}-admin-cli -i -- /bin/bash
-
-  pop_fn
 }
 
-function join_peers() {
+function join_channel_peers() {
   join_org_peers org1
   join_org_peers org2
 }
 
-# Copy the scripts/anchor_peers.sh to a remote volume
+function join_org_peers() {
-function push_anchor_peer_script() {
   local org=$1
+  push_fn "Joining ${org} peers to channel ${CHANNEL_NAME}"
 
-  tar cf - scripts/ | kubectl -n $NS exec -i -c main deploy/${org}-admin-cli -- tar xf - -C /var/hyperledger/fabric
+  # fetch the genesis block from an orderer.   (do we need to do this?  It's available locally...)
-}
 
-verify_result() {
+  # Join peers to channel
-  if [ $1 -ne 0 ]; then
+  join_channel_peer $org peer1
-    echo $2
+  join_channel_peer $org peer2
-    exit $1
-  fi
-}
-
-# Launch the anchor peer update script on a remote org admin CLI
-function invoke_anchor_peer_update() {
-  local org_num=$1
-  local peer_name=$2
-
-  kubectl exec \
-    -n $NS \
-    -c main \
-    deploy/org${org_num}-admin-cli \
-    -i \
-    /bin/bash -c "/var/hyperledger/fabric/scripts/set_anchor_peer.sh ${org_num} ${CHANNEL_NAME} ${peer_name}"
-
-  verify_result $? "Error updating anchor peer for org ${org_num}"
-}
-
-#
-# To update the anchor peers we will need to execute a script on each of the peer admin CLI containers.  These
-# commands can be individually piped into kubectl exec ... but it will be simpler if we transfer the anchor
-# peer update script over to the org volume and then trigger it from kubectl.
-#
-function update_anchor_peers() {
-  local peer_name=$1
-  push_fn "Updating anchor peers to ${peer_name}"
-
-  push_anchor_peer_script org1
-  push_anchor_peer_script org2
-
-  invoke_anchor_peer_update 1 ${peer_name}
-  invoke_anchor_peer_update 2 ${peer_name}
 
   pop_fn
 }
 
-function channel_up() {
+function join_channel_peer() {
+  local org=$1
+  local peer=$2
 
-  create_channel_MSP
+  FABRIC_CFG_PATH=${PWD}/config/${org} \
-  aggregate_channel_MSP
+  CORE_PEER_ADDRESS=${org}-${peer}.${DOMAIN}:443 \
-  launch_admin_CLIs
+  CORE_PEER_MSPCONFIGPATH=${TEMP_DIR}/enrollments/${org}/users/${org}admin/msp \
-
+  CORE_PEER_TLS_ROOTCERT_FILE=${TEMP_DIR}/channel-msp/peerOrganizations/${org}/msp/tlscacerts/tlsca-signcert.pem \
-  create_genesis_block
+    peer channel join \
-  join_peers
+      --blockpath ${TEMP_DIR}/genesis_block.pb \
-
+      --orderer   org0-orderer1.${DOMAIN} \
-  # peer1 was set as the anchor peer in configtx.yaml.  Setting this again will force an
+      --tls        \
-  # error to be returned from the channel up.  We might want to render the warning in
+      --cafile    ${TEMP_DIR}/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem
-  # this case to indicate that the call was made but had a nonzero exit. 
+}
-  # update_anchor_peers peer1
-}

test-network-k8s/scripts/fabric_CAs.sh

@@ -5,20 +5,12 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-function launch_CA() {
-  local yaml=$1
-  cat ${yaml} \
-    | sed 's,{{FABRIC_CONTAINER_REGISTRY}},'${FABRIC_CONTAINER_REGISTRY}',g' \
-    | sed 's,{{FABRIC_CA_VERSION}},'${FABRIC_CA_VERSION}',g' \
-    | kubectl -n $NS apply -f -
-}
-
 function launch_ECert_CAs() {
   push_fn "Launching Fabric CAs"
 
-  launch_CA kube/org0/org0-ca.yaml
+  apply_template kube/org0/org0-ca.yaml
-  launch_CA kube/org1/org1-ca.yaml
+  apply_template kube/org1/org1-ca.yaml
-  launch_CA kube/org2/org2-ca.yaml
+  apply_template kube/org2/org2-ca.yaml
 
   kubectl -n $NS rollout status deploy/org0-ca
   kubectl -n $NS rollout status deploy/org1-ca

test-network-k8s/scripts/kind.sh

@@ -87,8 +87,8 @@ nodes:
       - containerPort: 443
         hostPort: ${ingress_https_port}
         protocol: TCP
-networking:
+#networking:
-  kubeProxyMode: "ipvs"
+#  kubeProxyMode: "ipvs"
 
 # create a cluster with the local registry enabled in containerd
 containerdConfigPatches:
@@ -98,6 +98,12 @@ containerdConfigPatches:
 
 EOF
 
+  # workaround for https://github.com/hyperledger/fabric-samples/issues/550 - pods can not resolve external DNS
+  for node in $(kind get nodes);
+  do
+    docker exec "$node" sysctl net.ipv4.conf.all.route_localnet=1;
+  done
+
   pop_fn
 }
 

test-network-k8s/scripts/old_channel.sh

@@ -0,0 +1,186 @@
+#!/bin/bash
+#
+# Copyright IBM Corp All Rights Reserved
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+function create_channel_org_MSP() {
+  local org=$1
+  local org_type=$2
+  local ecert_ca=${org}-ca
+  
+  echo 'set -x
+ 
+  mkdir -p /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/cacerts
+  cp \
+    $FABRIC_CA_CLIENT_HOME/'${ecert_ca}'/rcaadmin/msp/cacerts/'${ecert_ca}'.pem \
+    /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/cacerts
+
+  mkdir -p /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/tlscacerts
+  cp \
+    /var/hyperledger/fabric/config/tls/ca.crt \
+    /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/tlscacerts/'${org}'-tls-ca.pem
+
+  echo "NodeOUs:
+    Enable: true
+    ClientOUIdentifier:
+      Certificate: cacerts/'${ecert_ca}'.pem
+      OrganizationalUnitIdentifier: client
+    PeerOUIdentifier:
+      Certificate: cacerts/'${ecert_ca}'.pem
+      OrganizationalUnitIdentifier: peer
+    AdminOUIdentifier:
+      Certificate: cacerts/'${ecert_ca}'.pem
+      OrganizationalUnitIdentifier: admin
+    OrdererOUIdentifier:
+      Certificate: cacerts/'${ecert_ca}'.pem
+      OrganizationalUnitIdentifier: orderer "> /var/hyperledger/fabric/organizations/'${org_type}'Organizations/'${org}'.example.com/msp/config.yaml
+      
+  ' | exec kubectl -n $NS exec deploy/${ecert_ca} -i -- /bin/sh
+}
+
+function create_channel_MSP() {
+  push_fn "Creating channel MSP"
+
+  create_channel_org_MSP org0 orderer 
+  create_channel_org_MSP org1 peer
+  create_channel_org_MSP org2 peer
+
+  pop_fn
+}
+
+function aggregate_channel_MSP() {
+  push_fn "Aggregating channel MSP"
+
+  rm -rf ./build/msp/
+  mkdir -p ./build/msp
+  
+  kubectl -n $NS exec deploy/org0-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/ordererOrganizations/org0.example.com/msp > build/msp/msp-org0.example.com.tgz
+  kubectl -n $NS exec deploy/org1-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/peerOrganizations/org1.example.com/msp > build/msp/msp-org1.example.com.tgz
+  kubectl -n $NS exec deploy/org2-ca -- tar zcvf - -C /var/hyperledger/fabric organizations/peerOrganizations/org2.example.com/msp > build/msp/msp-org2.example.com.tgz
+
+  kubectl -n $NS delete configmap msp-config || true
+  kubectl -n $NS create configmap msp-config --from-file=build/msp/
+
+  pop_fn
+}
+
+function create_genesis_block() {
+  push_fn "Creating channel \"${CHANNEL_NAME}\""
+
+  echo 'set -x
+  configtxgen -profile TwoOrgsApplicationGenesis -channelID '${CHANNEL_NAME}' -outputBlock genesis_block.pb
+  # configtxgen -inspectBlock genesis_block.pb
+  
+  osnadmin channel join --orderer-address org0-orderer1:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb
+  osnadmin channel join --orderer-address org0-orderer2:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb
+  osnadmin channel join --orderer-address org0-orderer3:9443 --channelID '${CHANNEL_NAME}' --config-block genesis_block.pb
+  
+  ' | exec kubectl -n $NS exec deploy/org0-admin-cli -i -- /bin/bash
+  
+  # todo: readiness / liveiness equivalent for channel ?    Needs a little bit to settle before peers can join. 
+  sleep 10
+
+  pop_fn
+}
+
+function join_org_peers() {
+  local org=$1
+  push_fn "Joining ${org} peers to channel \"${CHANNEL_NAME}\""
+
+  echo 'set -x
+  # Fetch the genesis block from an orderer
+  peer channel \
+    fetch oldest \
+    genesis_block.pb \
+    -c '${CHANNEL_NAME}' \
+    -o org0-orderer1:6050 \
+    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
+
+  # Join peer1 to the channel.
+  CORE_PEER_ADDRESS='${org}'-peer1:7051 \
+  peer channel \
+    join \
+    -b genesis_block.pb \
+    -o org0-orderer1:6050 \
+    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
+
+  # Join peer2 to the channel.
+  CORE_PEER_ADDRESS='${org}'-peer2:7051 \
+  peer channel \
+    join \
+    -b genesis_block.pb \
+    -o org0-orderer1:6050 \
+    --tls --cafile /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/msp/tlscacerts/org0-tls-ca.pem
+
+  ' | exec kubectl -n $NS exec deploy/${org}-admin-cli -i -- /bin/bash
+
+  pop_fn
+}
+
+function join_peers() {
+  join_org_peers org1
+  join_org_peers org2
+}
+
+# Copy the scripts/anchor_peers.sh to a remote volume
+function push_anchor_peer_script() {
+  local org=$1
+
+  tar cf - scripts/ | kubectl -n $NS exec -i -c main deploy/${org}-admin-cli -- tar xf - -C /var/hyperledger/fabric
+}
+
+verify_result() {
+  if [ $1 -ne 0 ]; then
+    echo $2
+    exit $1
+  fi
+}
+
+# Launch the anchor peer update script on a remote org admin CLI
+function invoke_anchor_peer_update() {
+  local org_num=$1
+  local peer_name=$2
+
+  kubectl exec \
+    -n $NS \
+    -c main \
+    deploy/org${org_num}-admin-cli \
+    -i \
+    /bin/bash -c "/var/hyperledger/fabric/scripts/set_anchor_peer.sh ${org_num} ${CHANNEL_NAME} ${peer_name}"
+
+  verify_result $? "Error updating anchor peer for org ${org_num}"
+}
+
+#
+# To update the anchor peers we will need to execute a script on each of the peer admin CLI containers.  These
+# commands can be individually piped into kubectl exec ... but it will be simpler if we transfer the anchor
+# peer update script over to the org volume and then trigger it from kubectl.
+#
+function update_anchor_peers() {
+  local peer_name=$1
+  push_fn "Updating anchor peers to ${peer_name}"
+
+  push_anchor_peer_script org1
+  push_anchor_peer_script org2
+
+  invoke_anchor_peer_update 1 ${peer_name}
+  invoke_anchor_peer_update 2 ${peer_name}
+
+  pop_fn
+}
+
+function channel_up() {
+
+  create_channel_MSP
+  aggregate_channel_MSP
+
+  create_genesis_block
+  join_peers
+
+  # peer1 was set as the anchor peer in configtx.yaml.  Setting this again will force an
+  # error to be returned from the channel up.  We might want to render the warning in
+  # this case to indicate that the call was made but had a nonzero exit. 
+  # update_anchor_peers peer1
+}

test-network-k8s/scripts/prereqs.sh

@@ -8,6 +8,8 @@
 # Double check that kind, kubectl, docker, and all required images are present.
 function check_prereqs() {
 
+  set +e
+
   ${CONTAINER_CLI} version > /dev/null
   if [[ $? -ne 0 ]]; then
     echo "No '${CONTAINER_CLI}' binary available?"
@@ -31,4 +33,27 @@ function check_prereqs() {
     echo "No 'jq' binary available? (https://stedolan.github.io/jq/)"
     exit 1
   fi
+
+  # Use the local fabric binaries if available.  If not, go get them.
+  bin/peer version &> /dev/null
+  if [[ $? -ne 0 ]]; then
+    echo "Downloading LATEST Fabric binaries and config"
+    curl -sSL https://raw.githubusercontent.com/hyperledger/fabric/main/scripts/bootstrap.sh | bash -s -- -s -d
+
+    # remove sample config files extracted by the installation script
+    rm config/configtx.yaml
+    rm config/core.yaml
+    rm config/orderer.yaml
+  fi
+
+  export PATH=bin:$PATH
+
+  # Double-check that the binary transfer was OK
+  peer version > /dev/null
+  if [[ $? -ne 0 ]]; then
+    log "No 'peer' binary available?"
+    exit 1
+  fi
+
+  set -e
 }

test-network-k8s/scripts/test_network.sh

@@ -5,20 +5,12 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-function launch() {
-  local yaml=$1
-  cat ${yaml} \
-    | sed 's,{{FABRIC_CONTAINER_REGISTRY}},'${FABRIC_CONTAINER_REGISTRY}',g' \
-    | sed 's,{{FABRIC_VERSION}},'${FABRIC_VERSION}',g' \
-    | kubectl -n $NS apply -f -
-}
-
 function launch_orderers() {
   push_fn "Launching orderers"
 
-  launch kube/org0/org0-orderer1.yaml
+  apply_template kube/org0/org0-orderer1.yaml
-  launch kube/org0/org0-orderer2.yaml
+  apply_template kube/org0/org0-orderer2.yaml
-  launch kube/org0/org0-orderer3.yaml
+  apply_template kube/org0/org0-orderer3.yaml
 
   kubectl -n $NS rollout status deploy/org0-orderer1
   kubectl -n $NS rollout status deploy/org0-orderer2
@@ -30,10 +22,10 @@ function launch_orderers() {
 function launch_peers() {
   push_fn "Launching peers"
 
-  launch kube/org1/org1-peer1.yaml
+  apply_template kube/org1/org1-peer1.yaml
-  launch kube/org1/org1-peer2.yaml
+  apply_template kube/org1/org1-peer2.yaml
-  launch kube/org2/org2-peer1.yaml
+  apply_template kube/org2/org2-peer1.yaml
-  launch kube/org2/org2-peer2.yaml
+  apply_template kube/org2/org2-peer2.yaml
 
   kubectl -n $NS rollout status deploy/org1-peer1
   kubectl -n $NS rollout status deploy/org1-peer2
@@ -43,155 +35,127 @@ function launch_peers() {
   pop_fn
 }
 
-function create_org0_local_MSP() {
+# todo: enroll org admin LOCALLY from the host OS
-  echo 'set -x
+#  fabric-ca-client register --id.name org1-admin --id.secret org1adminpw  --id.type admin   --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
+#  fabric-ca-client enroll --url https://org1-admin:org1adminpw@org1-ca  --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/server.key
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/config.yaml
+
+#  fabric-ca-client register --id.name org1-admin --id.secret org1adminpw  --id.type admin   --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
+#  fabric-ca-client enroll --url https://org1-admin:org1adminpw@org1-ca  --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/server.key
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/config.yaml
+
+#  fabric-ca-client register --id.name org2-admin --id.secret org2adminpw  --id.type admin   --url https://org2-ca --mspdir $FABRIC_CA_CLIENT_HOME/org2-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
+#  fabric-ca-client enroll --url https://org2-admin:org2adminpw@org2-ca  --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/keystore/server.key
+#  cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/config.yaml
+
+
+# Each network node needs a registration, enrollment, and MSP config.yaml
+function create_node_local_MSP() {
+  local node_type=$1
+  local org=$2
+  local node=$3
+  local csr_hosts=$4
+  local id_name=${org}-${node}
+  local id_secret=${node_type}pw
+  local ca_name=${org}-ca
+
+  cat <<EOF | kubectl -n $NS exec deploy/${ca_name} -i -- /bin/sh
+
+  set -x
   export FABRIC_CA_CLIENT_HOME=/var/hyperledger/fabric-ca-client
   export FABRIC_CA_CLIENT_TLS_CERTFILES=/var/hyperledger/fabric/config/tls/ca.crt
 
   # Each identity in the network needs a registration and enrollment.
-  fabric-ca-client register --id.name org0-orderer1 --id.secret ordererpw --id.type orderer --url https://org0-ca --mspdir $FABRIC_CA_CLIENT_HOME/org0-ca/rcaadmin/msp
+  fabric-ca-client register \
-  fabric-ca-client register --id.name org0-orderer2 --id.secret ordererpw --id.type orderer --url https://org0-ca --mspdir $FABRIC_CA_CLIENT_HOME/org0-ca/rcaadmin/msp
+    --id.name ${id_name} \
-  fabric-ca-client register --id.name org0-orderer3 --id.secret ordererpw --id.type orderer --url https://org0-ca --mspdir $FABRIC_CA_CLIENT_HOME/org0-ca/rcaadmin/msp
+    --id.secret ${id_secret} \
-  fabric-ca-client register --id.name org0-admin --id.secret org0adminpw  --id.type admin   --url https://org0-ca --mspdir $FABRIC_CA_CLIENT_HOME/org0-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
+    --id.type ${node_type} \
+    --url https://${ca_name} \
+    --mspdir /var/hyperledger/fabric-ca-client/${ca_name}/rcaadmin/msp
 
-  fabric-ca-client enroll --url https://org0-orderer1:ordererpw@org0-ca --csr.hosts org0-orderer1 --mspdir /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp
+  fabric-ca-client enroll \
-  fabric-ca-client enroll --url https://org0-orderer2:ordererpw@org0-ca --csr.hosts org0-orderer2 --mspdir /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/msp
+    --url https://${id_name}:${id_secret}@${ca_name} \
-  fabric-ca-client enroll --url https://org0-orderer3:ordererpw@org0-ca --csr.hosts org0-orderer3 --mspdir /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/msp
+    --csr.hosts ${csr_hosts} \
-  fabric-ca-client enroll --url https://org0-admin:org0adminpw@org0-ca --mspdir /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/users/Admin@org0.example.com/msp
+    --mspdir /var/hyperledger/fabric/organizations/${node_type}Organizations/${org}.example.com/${node_type}s/${id_name}.${org}.example.com/msp
-
-  # Create an MSP config.yaml (why is this not generated by the enrollment by fabric-ca-client?)
-  echo "NodeOUs:
-    Enable: true
-    ClientOUIdentifier:
-      Certificate: cacerts/org0-ca.pem
-      OrganizationalUnitIdentifier: client
-    PeerOUIdentifier:
-      Certificate: cacerts/org0-ca.pem
-      OrganizationalUnitIdentifier: peer
-    AdminOUIdentifier:
-      Certificate: cacerts/org0-ca.pem
-      OrganizationalUnitIdentifier: admin
-    OrdererOUIdentifier:
-      Certificate: cacerts/org0-ca.pem
-      OrganizationalUnitIdentifier: orderer" > /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp/config.yaml
-
-  cp /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer2.org0.example.com/msp/config.yaml
-  cp /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer1.org0.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/org0-orderer3.org0.example.com/msp/config.yaml
-  ' | exec kubectl -n $NS exec deploy/org0-ca -i -- /bin/sh
-}
-
-function create_org1_local_MSP() {
-
-  echo 'set -x
-  export FABRIC_CA_CLIENT_HOME=/var/hyperledger/fabric-ca-client
-  export FABRIC_CA_CLIENT_TLS_CERTFILES=/var/hyperledger/fabric/config/tls/ca.crt
-
-  # Each identity in the network needs a registration and enrollment.
-  fabric-ca-client register --id.name org1-peer1 --id.secret peerpw --id.type peer --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org1-peer2 --id.secret peerpw --id.type peer --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org1-admin --id.secret org1adminpw  --id.type admin   --url https://org1-ca --mspdir $FABRIC_CA_CLIENT_HOME/org1-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
-
-  fabric-ca-client enroll --url https://org1-peer1:peerpw@org1-ca --csr.hosts localhost,org1-peer1,org1-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp
-  fabric-ca-client enroll --url https://org1-peer2:peerpw@org1-ca --csr.hosts localhost,org1-peer2,org1-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer2.org1.example.com/msp
-  fabric-ca-client enroll --url https://org1-admin:org1adminpw@org1-ca  --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp
-
-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/keystore/server.key
 
   # Create local MSP config.yaml
   echo "NodeOUs:
     Enable: true
     ClientOUIdentifier:
-      Certificate: cacerts/org1-ca.pem
+      Certificate: cacerts/${org}-ca.pem
       OrganizationalUnitIdentifier: client
     PeerOUIdentifier:
-      Certificate: cacerts/org1-ca.pem
+      Certificate: cacerts/${org}-ca.pem
       OrganizationalUnitIdentifier: peer
     AdminOUIdentifier:
-      Certificate: cacerts/org1-ca.pem
+      Certificate: cacerts/${org}-ca.pem
       OrganizationalUnitIdentifier: admin
     OrdererOUIdentifier:
-      Certificate: cacerts/org1-ca.pem
+      Certificate: cacerts/${org}-ca.pem
-      OrganizationalUnitIdentifier: orderer" > /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml
+      OrganizationalUnitIdentifier: orderer" > /var/hyperledger/fabric/organizations/${node_type}Organizations/${org}.example.com/${node_type}s/${id_name}.${org}.example.com/msp/config.yaml
-
+EOF
-
-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer2.org1.example.com/msp/config.yaml
-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp/config.yaml
-  ' | exec kubectl -n $NS exec deploy/org1-ca -i -- /bin/sh
-
 }
 
-function create_org2_local_MSP() {
+function create_orderer_local_MSP() {
-  echo 'set -x
+  local org=$1
-  export FABRIC_CA_CLIENT_HOME=/var/hyperledger/fabric-ca-client
+  local orderer=$2
-  export FABRIC_CA_CLIENT_TLS_CERTFILES=/var/hyperledger/fabric/config/tls/ca.crt
+  local csr_hosts=${org}-${orderer}
 
-  # Each identity in the network needs a registration and enrollment.
+  create_node_local_MSP orderer $org $orderer $csr_hosts
-  fabric-ca-client register --id.name org2-peer1 --id.secret peerpw --id.type peer --url https://org2-ca --mspdir $FABRIC_CA_CLIENT_HOME/org2-ca/rcaadmin/msp
+}
-  fabric-ca-client register --id.name org2-peer2 --id.secret peerpw --id.type peer --url https://org2-ca --mspdir $FABRIC_CA_CLIENT_HOME/org2-ca/rcaadmin/msp
-  fabric-ca-client register --id.name org2-admin --id.secret org2adminpw  --id.type admin   --url https://org2-ca --mspdir $FABRIC_CA_CLIENT_HOME/org2-ca/rcaadmin/msp --id.attrs "hf.Registrar.Roles=client,hf.Registrar.Attributes=*,hf.Revoker=true,hf.GenCRL=true,admin=true:ecert,abac.init=true:ecert"
 
-  fabric-ca-client enroll --url https://org2-peer1:peerpw@org2-ca --csr.hosts localhost,org2-peer1,org2-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp
+function create_peer_local_MSP() {
-  fabric-ca-client enroll --url https://org2-peer2:peerpw@org2-ca --csr.hosts localhost,org2-peer2,org2-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer2.org2.example.com/msp
+  local org=$1
-  fabric-ca-client enroll --url https://org2-admin:org2adminpw@org2-ca  --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp
+  local peer=$2
+  local csr_hosts=localhost,${org}-${peer},${org}-peer-gateway-svc
 
-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/keystore/*_sk /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/keystore/server.key
+  create_node_local_MSP peer $org $peer $csr_hosts
-
-  # Create local MSP config.yaml
-  echo "NodeOUs:
-    Enable: true
-    ClientOUIdentifier:
-      Certificate: cacerts/org2-ca.pem
-      OrganizationalUnitIdentifier: client
-    PeerOUIdentifier:
-      Certificate: cacerts/org2-ca.pem
-      OrganizationalUnitIdentifier: peer
-    AdminOUIdentifier:
-      Certificate: cacerts/org2-ca.pem
-      OrganizationalUnitIdentifier: admin
-    OrdererOUIdentifier:
-      Certificate: cacerts/org2-ca.pem
-      OrganizationalUnitIdentifier: orderer" > /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp/config.yaml
-
-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer2.org2.example.com/msp/config.yaml
-  cp /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp/config.yaml /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/users/Admin@org2.example.com/msp/config.yaml
-  ' | exec kubectl -n $NS exec deploy/org2-ca -i -- /bin/sh
 }
 
 function create_local_MSP() {
   push_fn "Creating local node MSP"
 
-  create_org0_local_MSP
+  create_orderer_local_MSP org0 orderer1
-  create_org1_local_MSP
+  create_orderer_local_MSP org0 orderer2
-  create_org2_local_MSP
+  create_orderer_local_MSP org0 orderer3
-
+
-  pop_fn
+  create_peer_local_MSP org1 peer1
-}
+  create_peer_local_MSP org1 peer2
-
+
-# TLS certificates are isused by the CA's Issuer, stored in a Kube secret, and mounted into the pod at /var/hyperledger/fabric/config/tls.
+  create_peer_local_MSP org2 peer1
-# For consistency with the Fabric-CA guide, his function copies the orderer's TLS certs into the traditional Fabric MSP / folder structure.
+  create_peer_local_MSP org2 peer2
-function extract_orderer_tls_cert() {
-  local orderer=$1
-
-  echo 'set -x
-
-  mkdir -p /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/'${orderer}'.org0.example.com/tls/signcerts/
-
-  cp \
-    var/hyperledger/fabric/config/tls/tls.crt \
-    /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/'${orderer}'.org0.example.com/tls/signcerts/cert.pem
-
-  ' | exec kubectl -n $NS exec deploy/${orderer} -i -c main -- /bin/sh
-}
-
-function extract_orderer_tls_certs() {
-  push_fn "Extracting orderer TLS certs to local MSP folder"
-
-  extract_orderer_tls_cert org0-orderer1
-  extract_orderer_tls_cert org0-orderer2
-  extract_orderer_tls_cert org0-orderer3
 
   pop_fn
 }
+#
+## TLS certificates are isused by the CA's Issuer, stored in a Kube secret, and mounted into the pod at /var/hyperledger/fabric/config/tls.
+## For consistency with the Fabric-CA guide, his function copies the orderer's TLS certs into the traditional Fabric MSP / folder structure.
+#function extract_orderer_tls_cert() {
+#  local orderer=$1
+#
+#  echo 'set -x
+#
+#  mkdir -p /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/'${orderer}'.org0.example.com/tls/signcerts/
+#
+#  cp \
+#    var/hyperledger/fabric/config/tls/tls.crt \
+#    /var/hyperledger/fabric/organizations/ordererOrganizations/org0.example.com/orderers/'${orderer}'.org0.example.com/tls/signcerts/cert.pem
+#
+#  ' | exec kubectl -n $NS exec deploy/${orderer} -i -c main -- /bin/sh
+#}
+#
+#function extract_orderer_tls_certs() {
+#  push_fn "Extracting orderer TLS certs to local MSP folder"
+#
+#  extract_orderer_tls_cert org0-orderer1
+#  extract_orderer_tls_cert org0-orderer2
+#  extract_orderer_tls_cert org0-orderer3
+#
+#  pop_fn
+#}
 
 function network_up() {
 
@@ -213,17 +177,13 @@ function network_up() {
   launch_orderers
   launch_peers
 
-  extract_orderer_tls_certs
+#  extract_orderer_tls_certs
 }
 
 function stop_services() {
   push_fn "Stopping Fabric services"
 
-  # These pods are busy executing `sleep MAX_INT` and do not shut down very quickly...
+  kubectl -n $NS delete ingress --all
-#  kubectl -n $NS delete deployment/org0-admin-cli --grace-period=0 --force
-#  kubectl -n $NS delete deployment/org1-admin-cli --grace-period=0 --force
-#  kubectl -n $NS delete deployment/org2-admin-cli --grace-period=0 --force
-
   kubectl -n $NS delete deployment --all
   kubectl -n $NS delete pod --all
   kubectl -n $NS delete service --all
@@ -252,4 +212,6 @@ function scrub_org_volumes() {
 function network_down() {
   stop_services
   scrub_org_volumes
+
+  rm -rf $PWD/build
 }

test-network-k8s/scripts/utils.sh

@@ -25,6 +25,8 @@ function logging_init() {
 function exit_fn() {
   rc=$?
 
+  set +x
+
   # Write an error icon to the current logging statement.
   if [ "0" -ne $rc ]; then
     pop_fn $rc
@@ -73,3 +75,12 @@ function pop_fn() {
   echo "" >> ${LOG_FILE}
 }
 
+# Apply the current environment to a k8s template and apply to the cluster.
+function apply_template() {
+
+  echo "Applying template $1:"
+  cat $1 | envsubst
+
+  cat $1 | envsubst | kubectl -n $NS apply -f -
+}
+