From d1b3253cc56e1438667261e2c4931794af49cb64 Mon Sep 17 00:00:00 2001 From: Basil K Y Date: Fri, 26 Aug 2022 22:09:45 +0530 Subject: [PATCH] separate namespace for each organization (#817) * seperate namespace for each organization Signed-off-by: Basil K Y * rest server working Signed-off-by: Basil K Y * use single namespace by default, fix k8s buider Signed-off-by: Basil K Y * added ci test for testing k8s n/w on multiple namespaces Signed-off-by: Basil K Y * fix: access rest server via gateway Signed-off-by: Basil K Y Signed-off-by: Basil K Y --- ci/azure-pipelines.yml | 7 ++ .../{configtx.yaml => configtx-template.yaml} | 28 +++--- .../kube/fabric-builder-rolebinding.yaml | 2 +- test-network-k8s/kube/fabric-rest-sample.yaml | 6 +- test-network-k8s/kube/org0/org0-ca.yaml | 2 +- .../org0/org0-job-scrub-fabric-volumes.yaml | 32 +++++++ test-network-k8s/kube/org0/org0-orderer1.yaml | 4 +- test-network-k8s/kube/org0/org0-orderer2.yaml | 4 +- test-network-k8s/kube/org0/org0-orderer3.yaml | 4 +- test-network-k8s/kube/org1/org1-ca.yaml | 2 +- .../org1/org1-job-scrub-fabric-volumes.yaml | 32 +++++++ test-network-k8s/kube/org1/org1-peer1.yaml | 6 +- test-network-k8s/kube/org1/org1-peer2.yaml | 6 +- test-network-k8s/kube/org2/org2-ca.yaml | 2 +- .../org2-job-scrub-fabric-volumes.yaml} | 10 -- test-network-k8s/kube/org2/org2-peer1.yaml | 6 +- test-network-k8s/kube/org2/org2-peer2.yaml | 6 +- test-network-k8s/network | 3 + test-network-k8s/scripts/ccp-template.json | 6 +- test-network-k8s/scripts/chaincode.sh | 4 +- test-network-k8s/scripts/channel.sh | 16 ++-- test-network-k8s/scripts/fabric_CAs.sh | 41 ++++---- test-network-k8s/scripts/fabric_config.sh | 48 +++++----- test-network-k8s/scripts/rest_sample.sh | 22 +++-- test-network-k8s/scripts/test_network.sh | 94 ++++++++++--------- test-network-k8s/scripts/utils.sh | 2 +- 26 files changed, 238 insertions(+), 157 deletions(-) rename test-network-k8s/config/org0/{configtx.yaml => configtx-template.yaml} (92%) create mode 100644 test-network-k8s/kube/org0/org0-job-scrub-fabric-volumes.yaml create mode 100644 test-network-k8s/kube/org1/org1-job-scrub-fabric-volumes.yaml rename test-network-k8s/kube/{job-scrub-fabric-volumes.yaml => org2/org2-job-scrub-fabric-volumes.yaml} (65%) diff --git a/ci/azure-pipelines.yml b/ci/azure-pipelines.yml index f383347d..1fbb352c 100644 --- a/ci/azure-pipelines.yml +++ b/ci/azure-pipelines.yml @@ -147,6 +147,13 @@ jobs: CHAINCODE_NAME: basic CHAINCODE_LANGUAGE: java CHAINCODE_BUILDER: k8s + Multi-Namespace: + ORG0_NS: org0-namespace + ORG1_NS: org1-namespace + ORG2_NS: org2-namespace + CHAINCODE_NAME: basic + CHAINCODE_LANGUAGE: java + CHAINCODE_BUILDER: k8s steps: - template: templates/install-k8s-deps.yml diff --git a/test-network-k8s/config/org0/configtx.yaml b/test-network-k8s/config/org0/configtx-template.yaml similarity index 92% rename from test-network-k8s/config/org0/configtx.yaml rename to test-network-k8s/config/org0/configtx-template.yaml index 2981b261..5bbbf93f 100644 --- a/test-network-k8s/config/org0/configtx.yaml +++ b/test-network-k8s/config/org0/configtx-template.yaml @@ -25,7 +25,7 @@ Organizations: ID: OrdererMSP # MSPDir is the filesystem path which contains the MSP configuration - MSPDir: ../../build/channel-msp/ordererOrganizations/org0/msp + MSPDir: ./channel-msp/ordererOrganizations/org0/msp # Policies defines the set of policies at this level of the config tree # For organization policies, their canonical path is usually @@ -42,9 +42,9 @@ Organizations: Rule: "OR('OrdererMSP.admin')" OrdererEndpoints: - - org0-orderer1:6050 - - org0-orderer2:6050 - - org0-orderer3:6050 + - org0-orderer1.${ORG0_NS}.svc.cluster.local:6050 + - org0-orderer2.${ORG0_NS}.svc.cluster.local:6050 + - org0-orderer3.${ORG0_NS}.svc.cluster.local:6050 - &Org1 # DefaultOrg defines the organization which is used in the sampleconfig @@ -54,7 +54,7 @@ Organizations: # ID to load the MSP definition as ID: Org1MSP - MSPDir: ../../build/channel-msp/peerOrganizations/org1/msp + MSPDir: ./channel-msp/peerOrganizations/org1/msp # Policies defines the set of policies at this level of the config tree # For organization policies, their canonical path is usually @@ -78,7 +78,7 @@ Organizations: # AnchorPeers defines the location of peers which can be used # for cross org gossip communication. Note, this value is only # encoded in the genesis block in the Application section context - - Host: org1-peer1 + - Host: org1-peer1.${ORG1_NS}.svc.cluster.local Port: 7051 - &Org2 @@ -89,7 +89,7 @@ Organizations: # ID to load the MSP definition as ID: Org2MSP - MSPDir: ../../build/channel-msp/peerOrganizations/org2/msp + MSPDir: ./channel-msp/peerOrganizations/org2/msp # Policies defines the set of policies at this level of the config tree # For organization policies, their canonical path is usually @@ -112,7 +112,7 @@ Organizations: # AnchorPeers defines the location of peers which can be used # for cross org gossip communication. Note, this value is only # encoded in the genesis block in the Application section context - - Host: org2-peer1 + - Host: org2-peer1.${ORG2_NS}.svc.cluster.local Port: 7051 ################################################################################ @@ -224,16 +224,16 @@ Orderer: &OrdererDefaults Consenters: - Host: org0-orderer1 Port: 6050 - ClientTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem - ServerTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem + ClientTLSCert: ./channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem + ServerTLSCert: ./channel-msp/ordererOrganizations/org0/orderers/org0-orderer1/tls/signcerts/tls-cert.pem - Host: org0-orderer2 Port: 6050 - ClientTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer2/tls/signcerts/tls-cert.pem - ServerTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer2/tls/signcerts/tls-cert.pem + ClientTLSCert: ./channel-msp/ordererOrganizations/org0/orderers/org0-orderer2/tls/signcerts/tls-cert.pem + ServerTLSCert: ./channel-msp/ordererOrganizations/org0/orderers/org0-orderer2/tls/signcerts/tls-cert.pem - Host: org0-orderer3 Port: 6050 - ClientTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer3/tls/signcerts/tls-cert.pem - ServerTLSCert: ../../build/channel-msp/ordererOrganizations/org0/orderers/org0-orderer3/tls/signcerts/tls-cert.pem + ClientTLSCert: ./channel-msp/ordererOrganizations/org0/orderers/org0-orderer3/tls/signcerts/tls-cert.pem + ServerTLSCert: ./channel-msp/ordererOrganizations/org0/orderers/org0-orderer3/tls/signcerts/tls-cert.pem # Options to be specified for all the etcd/raft nodes. The values here diff --git a/test-network-k8s/kube/fabric-builder-rolebinding.yaml b/test-network-k8s/kube/fabric-builder-rolebinding.yaml index a29a9278..057ccb0f 100644 --- a/test-network-k8s/kube/fabric-builder-rolebinding.yaml +++ b/test-network-k8s/kube/fabric-builder-rolebinding.yaml @@ -13,6 +13,6 @@ roleRef: kind: Role name: fabric-builder-role subjects: - - namespace: ${NS} + - namespace: ${ORG1_NS} kind: ServiceAccount name: default \ No newline at end of file diff --git a/test-network-k8s/kube/fabric-rest-sample.yaml b/test-network-k8s/kube/fabric-rest-sample.yaml index b9a75af4..8c3847e6 100644 --- a/test-network-k8s/kube/fabric-rest-sample.yaml +++ b/test-network-k8s/kube/fabric-rest-sample.yaml @@ -11,7 +11,7 @@ metadata: data: HLF_CONNECTION_PROFILE_ORG1: | { - "name": "${NS}-org1", + "name": "Org1", "version": "1.0.0", "client": { "organization": "Org1", @@ -87,7 +87,7 @@ data: -----END PRIVATE KEY----- HLF_CONNECTION_PROFILE_ORG2: | { - "name": "${NS}-org2", + "name": "Org2", "version": "1.0.0", "client": { "organization": "Org2", @@ -112,7 +112,7 @@ data: }, "peers": { "org2-peers": { - "url": "grpcs://org2-peer-gateway-svc:7051", + "url": "org2-peer-gateway-svc:7051", "tlsCACerts": { "pem": "-----BEGIN CERTIFICATE-----\\nMIICKDCCAc6gAwIBAgIUJJ4wGOSCfw8XOOIx29o67wBpFB4wCgYIKoZIzj0EAwIw\\naDELMAkGA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMRQwEgYDVQQK\\nEwtIeXBlcmxlZGdlcjEPMA0GA1UECxMGRmFicmljMRkwFwYDVQQDExBmYWJyaWMt\\nY2Etc2VydmVyMB4XDTIxMDkyMDExNDEwMFoXDTM2MDkxNjExNDEwMFowaDELMAkG\\nA1UEBhMCVVMxFzAVBgNVBAgTDk5vcnRoIENhcm9saW5hMRQwEgYDVQQKEwtIeXBl\\ncmxlZGdlcjEPMA0GA1UECxMGRmFicmljMRkwFwYDVQQDExBmYWJyaWMtY2Etc2Vy\\ndmVyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEyzGJLZX6pe59QAIBacjfzU4I\\nHezBYLyEu4ySpFx4xwxNLE4BWqLhB1VaOuenSQATM8pmSAy7i1830oM9elKWK6NW\\nMFQwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYDVR0OBBYE\\nFEoAAhmjq/3M8CFPc7N8SL53erL5MA8GA1UdEQQIMAaHBH8AAAEwCgYIKoZIzj0E\\nAwIDSAAwRQIhAJQ5PJOT4Gg8oiBU2KthMPkZqOLeu3Li4S3yBpLFgbsgAiB960P2\\nXPMu3HLoNXrktYOL9JzWlGyYRSPAnkap5Bsj0w==\\n-----END CERTIFICATE-----\\n" }, diff --git a/test-network-k8s/kube/org0/org0-ca.yaml b/test-network-k8s/kube/org0/org0-ca.yaml index 43791b91..87c081ac 100644 --- a/test-network-k8s/kube/org0/org0-ca.yaml +++ b/test-network-k8s/kube/org0/org0-ca.yaml @@ -16,7 +16,7 @@ spec: dnsNames: - localhost - org0-ca - - org0-ca.${NS}.svc.cluster.local + - org0-ca.${ORG0_NS}.svc.cluster.local - org0-ca.${DOMAIN} ipAddresses: - 127.0.0.1 diff --git a/test-network-k8s/kube/org0/org0-job-scrub-fabric-volumes.yaml b/test-network-k8s/kube/org0/org0-job-scrub-fabric-volumes.yaml new file mode 100644 index 00000000..92847115 --- /dev/null +++ b/test-network-k8s/kube/org0/org0-job-scrub-fabric-volumes.yaml @@ -0,0 +1,32 @@ +# +# Copyright IBM Corp. All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: job-scrub-fabric-volumes +spec: + backoffLimit: 0 + completions: 1 + template: + metadata: + name: job-scrub-fabric-volumes + spec: + restartPolicy: "Never" + containers: + - name: main + image: busybox:latest + command: + - sh + - -c + - "rm -rvf /mnt/fabric-*/*" + volumeMounts: + - name: fabric-org0-volume + mountPath: /mnt/fabric-org0 + volumes: + - name: fabric-org0-volume + persistentVolumeClaim: + claimName: fabric-org0 diff --git a/test-network-k8s/kube/org0/org0-orderer1.yaml b/test-network-k8s/kube/org0/org0-orderer1.yaml index f8bf9c36..108d1b0b 100644 --- a/test-network-k8s/kube/org0/org0-orderer1.yaml +++ b/test-network-k8s/kube/org0/org0-orderer1.yaml @@ -9,7 +9,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: org0-orderer1-tls-cert - namespace: ${NS} + namespace: ${ORG0_NS} spec: isCA: false privateKey: @@ -18,7 +18,7 @@ spec: dnsNames: - localhost - org0-orderer1 - - org0-orderer1.${NS}.svc.cluster.local + - org0-orderer1.${ORG0_NS}.svc.cluster.local - org0-orderer1.${DOMAIN} - org0-orderer1-admin.${DOMAIN} ipAddresses: diff --git a/test-network-k8s/kube/org0/org0-orderer2.yaml b/test-network-k8s/kube/org0/org0-orderer2.yaml index fa96f90e..043b13c8 100644 --- a/test-network-k8s/kube/org0/org0-orderer2.yaml +++ b/test-network-k8s/kube/org0/org0-orderer2.yaml @@ -9,7 +9,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: org0-orderer2-tls-cert - namespace: ${NS} + namespace: ${ORG0_NS} spec: isCA: false privateKey: @@ -18,7 +18,7 @@ spec: dnsNames: - localhost - org0-orderer2 - - org0-orderer2.${NS}.svc.cluster.local + - org0-orderer2.${ORG0_NS}.svc.cluster.local - org0-orderer2.${DOMAIN} - org0-orderer2-admin.${DOMAIN} ipAddresses: diff --git a/test-network-k8s/kube/org0/org0-orderer3.yaml b/test-network-k8s/kube/org0/org0-orderer3.yaml index 3391f6a8..3e42a739 100644 --- a/test-network-k8s/kube/org0/org0-orderer3.yaml +++ b/test-network-k8s/kube/org0/org0-orderer3.yaml @@ -9,7 +9,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: org0-orderer3-tls-cert - namespace: ${NS} + namespace: ${ORG0_NS} spec: isCA: false privateKey: @@ -18,7 +18,7 @@ spec: dnsNames: - localhost - org0-orderer3 - - org0-orderer3.${NS}.svc.cluster.local + - org0-orderer3.${ORG0_NS}.svc.cluster.local - org0-orderer3.${DOMAIN} - org0-orderer3-admin.${DOMAIN} ipAddresses: diff --git a/test-network-k8s/kube/org1/org1-ca.yaml b/test-network-k8s/kube/org1/org1-ca.yaml index d8336e33..01515226 100644 --- a/test-network-k8s/kube/org1/org1-ca.yaml +++ b/test-network-k8s/kube/org1/org1-ca.yaml @@ -16,7 +16,7 @@ spec: dnsNames: - localhost - org1-ca - - org1-ca.${NS}.svc.cluster.local + - org1-ca.${ORG1_NS}.svc.cluster.local - org1-ca.${DOMAIN} ipAddresses: - 127.0.0.1 diff --git a/test-network-k8s/kube/org1/org1-job-scrub-fabric-volumes.yaml b/test-network-k8s/kube/org1/org1-job-scrub-fabric-volumes.yaml new file mode 100644 index 00000000..43d66c4a --- /dev/null +++ b/test-network-k8s/kube/org1/org1-job-scrub-fabric-volumes.yaml @@ -0,0 +1,32 @@ +# +# Copyright IBM Corp. All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: job-scrub-fabric-volumes +spec: + backoffLimit: 0 + completions: 1 + template: + metadata: + name: job-scrub-fabric-volumes + spec: + restartPolicy: "Never" + containers: + - name: main + image: busybox:latest + command: + - sh + - -c + - "rm -rvf /mnt/fabric-*/*" + volumeMounts: + - name: fabric-org1-volume + mountPath: /mnt/fabric-org1 + volumes: + - name: fabric-org1-volume + persistentVolumeClaim: + claimName: fabric-org1 diff --git a/test-network-k8s/kube/org1/org1-peer1.yaml b/test-network-k8s/kube/org1/org1-peer1.yaml index dbb4a9d3..caa83e12 100644 --- a/test-network-k8s/kube/org1/org1-peer1.yaml +++ b/test-network-k8s/kube/org1/org1-peer1.yaml @@ -8,7 +8,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: org1-peer1-tls-cert - namespace: ${NS} + namespace: ${ORG1_NS} spec: isCA: false privateKey: @@ -17,7 +17,7 @@ spec: dnsNames: - localhost - org1-peer1 - - org1-peer1.${NS}.svc.cluster.local + - org1-peer1.${ORG1_NS}.svc.cluster.local - org1-peer1.${DOMAIN} - org1-peer-gateway-svc - org1-peer-gateway-svc.${DOMAIN} @@ -46,7 +46,7 @@ data: CORE_PEER_CHAINCODELISTENADDRESS: 0.0.0.0:7052 # bootstrap peer is the other peer in the same org CORE_PEER_GOSSIP_BOOTSTRAP: org1-peer2:7051 - CORE_PEER_GOSSIP_EXTERNALENDPOINT: org1-peer1:7051 + CORE_PEER_GOSSIP_EXTERNALENDPOINT: org1-peer1.${ORG1_NS}.svc.cluster.local:7051 CORE_PEER_LOCALMSPID: Org1MSP CORE_PEER_MSPCONFIGPATH: /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp CORE_OPERATIONS_LISTENADDRESS: 0.0.0.0:9443 diff --git a/test-network-k8s/kube/org1/org1-peer2.yaml b/test-network-k8s/kube/org1/org1-peer2.yaml index e485a558..f911618c 100644 --- a/test-network-k8s/kube/org1/org1-peer2.yaml +++ b/test-network-k8s/kube/org1/org1-peer2.yaml @@ -8,7 +8,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: org1-peer2-tls-cert - namespace: ${NS} + namespace: ${ORG1_NS} spec: isCA: false privateKey: @@ -18,7 +18,7 @@ spec: - localhost - org1-peer2 - org1-peer-gateway-svc - - org1-peer2.${NS}.svc.cluster.local + - org1-peer2.${ORG1_NS}.svc.cluster.local - org1-peer2.${DOMAIN} ipAddresses: - 127.0.0.1 @@ -46,7 +46,7 @@ data: CORE_PEER_CHAINCODELISTENADDRESS: 0.0.0.0:7052 # bootstrap peer is the other peer in the same org CORE_PEER_GOSSIP_BOOTSTRAP: org1-peer1:7051 - CORE_PEER_GOSSIP_EXTERNALENDPOINT: org1-peer2:7051 + CORE_PEER_GOSSIP_EXTERNALENDPOINT: org1-peer2.${ORG1_NS}.svc.cluster.local:7051 CORE_PEER_LOCALMSPID: Org1MSP CORE_PEER_MSPCONFIGPATH: /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer2.org1.example.com/msp CORE_OPERATIONS_LISTENADDRESS: 0.0.0.0:9443 diff --git a/test-network-k8s/kube/org2/org2-ca.yaml b/test-network-k8s/kube/org2/org2-ca.yaml index ba07eed1..53ce387a 100644 --- a/test-network-k8s/kube/org2/org2-ca.yaml +++ b/test-network-k8s/kube/org2/org2-ca.yaml @@ -16,7 +16,7 @@ spec: dnsNames: - localhost - org2-ca - - org2-ca.${NS}.svc.cluster.local + - org2-ca.${ORG2_NS}.svc.cluster.local - org2-ca.${DOMAIN} ipAddresses: - 127.0.0.1 diff --git a/test-network-k8s/kube/job-scrub-fabric-volumes.yaml b/test-network-k8s/kube/org2/org2-job-scrub-fabric-volumes.yaml similarity index 65% rename from test-network-k8s/kube/job-scrub-fabric-volumes.yaml rename to test-network-k8s/kube/org2/org2-job-scrub-fabric-volumes.yaml index ce2b4f48..b56677f2 100644 --- a/test-network-k8s/kube/job-scrub-fabric-volumes.yaml +++ b/test-network-k8s/kube/org2/org2-job-scrub-fabric-volumes.yaml @@ -24,19 +24,9 @@ spec: - -c - "rm -rvf /mnt/fabric-*/*" volumeMounts: - - name: fabric-org0-volume - mountPath: /mnt/fabric-org0 - - name: fabric-org1-volume - mountPath: /mnt/fabric-org1 - name: fabric-org2-volume mountPath: /mnt/fabric-org2 volumes: - - name: fabric-org0-volume - persistentVolumeClaim: - claimName: fabric-org0 - - name: fabric-org1-volume - persistentVolumeClaim: - claimName: fabric-org1 - name: fabric-org2-volume persistentVolumeClaim: claimName: fabric-org2 diff --git a/test-network-k8s/kube/org2/org2-peer1.yaml b/test-network-k8s/kube/org2/org2-peer1.yaml index 03c19fce..610c2aee 100644 --- a/test-network-k8s/kube/org2/org2-peer1.yaml +++ b/test-network-k8s/kube/org2/org2-peer1.yaml @@ -8,7 +8,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: org2-peer1-tls-cert - namespace: ${NS} + namespace: ${ORG2_NS} spec: isCA: false privateKey: @@ -17,7 +17,7 @@ spec: dnsNames: - localhost - org2-peer1 - - org2-peer1.${NS}.svc.cluster.local + - org2-peer1.${ORG2_NS}.svc.cluster.local - org2-peer1.${DOMAIN} - org2-peer-gateway-svc - org2-peer-gateway-svc.${DOMAIN} @@ -46,7 +46,7 @@ data: CORE_PEER_CHAINCODELISTENADDRESS: 0.0.0.0:7052 # bootstrap peer is the other peer in the same org CORE_PEER_GOSSIP_BOOTSTRAP: org2-peer2:7051 - CORE_PEER_GOSSIP_EXTERNALENDPOINT: org2-peer1:7051 + CORE_PEER_GOSSIP_EXTERNALENDPOINT: org2-peer1.${ORG2_NS}.svc.cluster.local:7051 CORE_PEER_LOCALMSPID: Org2MSP CORE_PEER_MSPCONFIGPATH: /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer1.org2.example.com/msp CORE_OPERATIONS_LISTENADDRESS: 0.0.0.0:9443 diff --git a/test-network-k8s/kube/org2/org2-peer2.yaml b/test-network-k8s/kube/org2/org2-peer2.yaml index e5584ac8..8ba1958b 100644 --- a/test-network-k8s/kube/org2/org2-peer2.yaml +++ b/test-network-k8s/kube/org2/org2-peer2.yaml @@ -8,7 +8,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: org2-peer2-tls-cert - namespace: ${NS} + namespace: ${ORG2_NS} spec: isCA: false privateKey: @@ -18,7 +18,7 @@ spec: - localhost - org2-peer2 - org2-peer-gateway-svc - - org2-peer2.${NS}.svc.cluster.local + - org2-peer2.${ORG2_NS}.svc.cluster.local - org2-peer2.${DOMAIN} ipAddresses: - 127.0.0.1 @@ -45,7 +45,7 @@ data: CORE_PEER_CHAINCODELISTENADDRESS: 0.0.0.0:7052 # bootstrap peer is the other peer in the same org CORE_PEER_GOSSIP_BOOTSTRAP: org2-peer1:7051 - CORE_PEER_GOSSIP_EXTERNALENDPOINT: org2-peer2:7051 + CORE_PEER_GOSSIP_EXTERNALENDPOINT: org2-peer2.${ORG2_NS}.svc.cluster.local:7051 CORE_PEER_LOCALMSPID: Org2MSP CORE_PEER_MSPCONFIGPATH: /var/hyperledger/fabric/organizations/peerOrganizations/org2.example.com/peers/org2-peer2.org2.example.com/msp CORE_OPERATIONS_LISTENADDRESS: 0.0.0.0:9443 diff --git a/test-network-k8s/network b/test-network-k8s/network index e696864a..20ee8595 100755 --- a/test-network-k8s/network +++ b/test-network-k8s/network @@ -41,6 +41,9 @@ context NETWORK_NAME test-network context CLUSTER_NAME kind context KUBE_NAMESPACE ${NETWORK_NAME} context NS ${KUBE_NAMESPACE} +context ORG0_NS ${NS} +context ORG1_NS ${NS} +context ORG2_NS ${NS} context DOMAIN localho.st context CHANNEL_NAME mychannel context ORDERER_TIMEOUT 10s # see https://github.com/hyperledger/fabric/issues/3372 diff --git a/test-network-k8s/scripts/ccp-template.json b/test-network-k8s/scripts/ccp-template.json index e3e2b290..b52b773a 100755 --- a/test-network-k8s/scripts/ccp-template.json +++ b/test-network-k8s/scripts/ccp-template.json @@ -24,13 +24,13 @@ }, "peers": { "org${ORG}-peers": { - "url": "grpcs://org${ORG}-peer-gateway-svc:7051", + "url": "grpcs://org${ORG}-peer1.${NS}.svc.cluster.local:7051", "tlsCACerts": { "pem": "${PEERPEM}" }, "grpcOptions": { - "ssl-target-name-override": "org${ORG}-peer-gateway-svc", - "hostnameOverride": "org${ORG}-peer-gateway-svc" + "ssl-target-name-override": "org${ORG}-peer1.${NS}.svc.cluster.local", + "hostnameOverride": "org${ORG}-peer1.${NS}.svc.cluster.local" } } }, diff --git a/test-network-k8s/scripts/chaincode.sh b/test-network-k8s/scripts/chaincode.sh index 5210a225..b8026ebe 100755 --- a/test-network-k8s/scripts/chaincode.sh +++ b/test-network-k8s/scripts/chaincode.sh @@ -299,9 +299,9 @@ function launch_chaincode_service() { | sed 's,{{CHAINCODE_ID}},'${cc_id}',g' \ | sed 's,{{CHAINCODE_IMAGE}},'${cc_image}',g' \ | sed 's,{{PEER_NAME}},'${peer}',g' \ - | exec kubectl -n $NS apply -f - + | exec kubectl -n $ORG1_NS apply -f - - kubectl -n $NS rollout status deploy/${org}${peer}-ccaas-${cc_name} + kubectl -n $ORG1_NS rollout status deploy/${org}${peer}-ccaas-${cc_name} pop_fn } diff --git a/test-network-k8s/scripts/channel.sh b/test-network-k8s/scripts/channel.sh index 080d4632..a510b7b7 100644 --- a/test-network-k8s/scripts/channel.sh +++ b/test-network-k8s/scripts/channel.sh @@ -146,9 +146,9 @@ EOF function create_channel_MSP() { push_fn "Creating channel MSP" - create_channel_org_MSP org0 orderer - create_channel_org_MSP org1 peer - create_channel_org_MSP org2 peer + create_channel_org_MSP org0 orderer $ORG0_NS + create_channel_org_MSP org1 peer $ORG1_NS + create_channel_org_MSP org2 peer $ORG2_NS extract_orderer_tls_cert org0 orderer1 extract_orderer_tls_cert org0 orderer2 @@ -160,6 +160,7 @@ function create_channel_MSP() { function create_channel_org_MSP() { local org=$1 local type=$2 + local ns=$3 local ca_name=${org}-ca ORG_MSP_DIR=${TEMP_DIR}/channel-msp/${type}Organizations/${org}/msp @@ -175,7 +176,7 @@ function create_channel_org_MSP() { > ${ORG_MSP_DIR}/cacerts/ca-signcert.pem # extract the CA's TLS CA certificate from the cert-manager secret - kubectl -n $NS get secret ${ca_name}-tls-cert -o json \ + kubectl -n $ns get secret ${ca_name}-tls-cert -o json \ | jq -r .data.\"ca.crt\" \ | base64 -d \ > ${ORG_MSP_DIR}/tlscacerts/tlsca-signcert.pem @@ -188,13 +189,14 @@ function create_channel_org_MSP() { function extract_orderer_tls_cert() { local org=$1 local orderer=$2 + local ns=$ORG0_NS echo "Extracting TLS cert for $org $orderer" ORDERER_TLS_DIR=${TEMP_DIR}/channel-msp/ordererOrganizations/${org}/orderers/${org}-${orderer}/tls mkdir -p $ORDERER_TLS_DIR/signcerts - kubectl -n $NS get secret ${org}-${orderer}-tls-cert -o json \ + kubectl -n $ns get secret ${org}-${orderer}-tls-cert -o json \ | jq -r .data.\"tls.crt\" \ | base64 -d \ > ${ORDERER_TLS_DIR}/signcerts/tls-cert.pem @@ -202,8 +204,8 @@ function extract_orderer_tls_cert() { function create_genesis_block() { push_fn "Creating channel genesis block" - - FABRIC_CFG_PATH=${PWD}/config/org0 \ + cat ${PWD}/config/org0/configtx-template.yaml | envsubst > ${TEMP_DIR}/configtx.yaml + FABRIC_CFG_PATH=${TEMP_DIR} \ configtxgen \ -profile TwoOrgsApplicationGenesis \ -channelID $CHANNEL_NAME \ diff --git a/test-network-k8s/scripts/fabric_CAs.sh b/test-network-k8s/scripts/fabric_CAs.sh index a47b96b4..17b9979c 100755 --- a/test-network-k8s/scripts/fabric_CAs.sh +++ b/test-network-k8s/scripts/fabric_CAs.sh @@ -8,13 +8,13 @@ function launch_ECert_CAs() { push_fn "Launching Fabric CAs" - apply_template kube/org0/org0-ca.yaml - apply_template kube/org1/org1-ca.yaml - apply_template kube/org2/org2-ca.yaml + apply_template kube/org0/org0-ca.yaml $ORG0_NS + apply_template kube/org1/org1-ca.yaml $ORG1_NS + apply_template kube/org2/org2-ca.yaml $ORG2_NS - kubectl -n $NS rollout status deploy/org0-ca - kubectl -n $NS rollout status deploy/org1-ca - kubectl -n $NS rollout status deploy/org2-ca + kubectl -n $ORG0_NS rollout status deploy/org0-ca + kubectl -n $ORG1_NS rollout status deploy/org1-ca + kubectl -n $ORG2_NS rollout status deploy/org2-ca # todo: this papers over a nasty bug whereby the CAs are ready, but sporadically refuse connections after a down / up sleep 5 @@ -28,23 +28,28 @@ function init_tls_cert_issuers() { # Create a self-signing certificate issuer / root TLS certificate for the blockchain. # TODO : Bring-Your-Own-Key - allow the network bootstrap to read an optional ECDSA key pair for the TLS trust root CA. - kubectl -n $NS apply -f kube/root-tls-cert-issuer.yaml - kubectl -n $NS wait --timeout=30s --for=condition=Ready issuer/root-tls-cert-issuer + kubectl -n $ORG0_NS apply -f kube/root-tls-cert-issuer.yaml + kubectl -n $ORG0_NS wait --timeout=30s --for=condition=Ready issuer/root-tls-cert-issuer + kubectl -n $ORG1_NS apply -f kube/root-tls-cert-issuer.yaml + kubectl -n $ORG1_NS wait --timeout=30s --for=condition=Ready issuer/root-tls-cert-issuer + kubectl -n $ORG2_NS apply -f kube/root-tls-cert-issuer.yaml + kubectl -n $ORG2_NS wait --timeout=30s --for=condition=Ready issuer/root-tls-cert-issuer # Use the self-signing issuer to generate three Issuers, one for each org. - kubectl -n $NS apply -f kube/org0/org0-tls-cert-issuer.yaml - kubectl -n $NS apply -f kube/org1/org1-tls-cert-issuer.yaml - kubectl -n $NS apply -f kube/org2/org2-tls-cert-issuer.yaml + kubectl -n $ORG0_NS apply -f kube/org0/org0-tls-cert-issuer.yaml + kubectl -n $ORG1_NS apply -f kube/org1/org1-tls-cert-issuer.yaml + kubectl -n $ORG2_NS apply -f kube/org2/org2-tls-cert-issuer.yaml - kubectl -n $NS wait --timeout=30s --for=condition=Ready issuer/org0-tls-cert-issuer - kubectl -n $NS wait --timeout=30s --for=condition=Ready issuer/org1-tls-cert-issuer - kubectl -n $NS wait --timeout=30s --for=condition=Ready issuer/org2-tls-cert-issuer + kubectl -n $ORG0_NS wait --timeout=30s --for=condition=Ready issuer/org0-tls-cert-issuer + kubectl -n $ORG1_NS wait --timeout=30s --for=condition=Ready issuer/org1-tls-cert-issuer + kubectl -n $ORG2_NS wait --timeout=30s --for=condition=Ready issuer/org2-tls-cert-issuer pop_fn } function enroll_bootstrap_ECert_CA_user() { local org=$1 + local ns=$2 # Determine the CA information and TLS certificate CA_NAME=${org}-ca @@ -53,7 +58,7 @@ function enroll_bootstrap_ECert_CA_user() { # Read the CA's TLS certificate from the cert-manager CA secret echo "retrieving ${CA_NAME} TLS root cert" - kubectl -n $NS get secret ${CA_NAME}-tls-cert -o json \ + kubectl -n $ns get secret ${CA_NAME}-tls-cert -o json \ | jq -r .data.\"ca.crt\" \ | base64 -d \ > ${CA_DIR}/tlsca-cert.pem @@ -68,9 +73,9 @@ function enroll_bootstrap_ECert_CA_user() { function enroll_bootstrap_ECert_CA_users() { push_fn "Enrolling bootstrap ECert CA users" - enroll_bootstrap_ECert_CA_user org0 - enroll_bootstrap_ECert_CA_user org1 - enroll_bootstrap_ECert_CA_user org2 + enroll_bootstrap_ECert_CA_user org0 $ORG0_NS + enroll_bootstrap_ECert_CA_user org1 $ORG1_NS + enroll_bootstrap_ECert_CA_user org2 $ORG2_NS pop_fn } \ No newline at end of file diff --git a/test-network-k8s/scripts/fabric_config.sh b/test-network-k8s/scripts/fabric_config.sh index 4bc31779..fb478490 100755 --- a/test-network-k8s/scripts/fabric_config.sh +++ b/test-network-k8s/scripts/fabric_config.sh @@ -6,19 +6,21 @@ # function init_namespace() { - push_fn "Creating namespace \"$NS\"" - - kubectl create namespace $NS || true - - pop_fn + local namespaces=$(echo "$ORG0_NS $ORG1_NS $ORG2_NS" | xargs -n1 | sort -u) + for ns in $namespaces; do + push_fn "Creating namespace \"$ns\"" + kubectl create namespace $ns || true + pop_fn + done } function delete_namespace() { - push_fn "Deleting namespace \"$NS\"" - - kubectl delete namespace $NS || true - - pop_fn + local namespaces=$(echo "$ORG0_NS $ORG1_NS $ORG2_NS" | xargs -n1 | sort -u) + for ns in $namespaces; do + push_fn "Deleting namespace \"$ns\"" + kubectl delete namespace $ns || true + pop_fn + done } function init_storage_volumes() { @@ -37,9 +39,9 @@ function init_storage_volumes() { exit 1 fi - cat kube/pvc-fabric-org0.yaml | envsubst | kubectl -n $NS create -f - || true - cat kube/pvc-fabric-org1.yaml | envsubst | kubectl -n $NS create -f - || true - cat kube/pvc-fabric-org2.yaml | envsubst | kubectl -n $NS create -f - || true + cat kube/pvc-fabric-org0.yaml | envsubst | kubectl -n $ORG0_NS create -f - || true + cat kube/pvc-fabric-org1.yaml | envsubst | kubectl -n $ORG1_NS create -f - || true + cat kube/pvc-fabric-org2.yaml | envsubst | kubectl -n $ORG2_NS create -f - || true pop_fn } @@ -47,13 +49,13 @@ function init_storage_volumes() { function load_org_config() { push_fn "Creating fabric config maps" - kubectl -n $NS delete configmap org0-config || true - kubectl -n $NS delete configmap org1-config || true - kubectl -n $NS delete configmap org2-config || true + kubectl -n $ORG0_NS delete configmap org0-config || true + kubectl -n $ORG1_NS delete configmap org1-config || true + kubectl -n $ORG2_NS delete configmap org2-config || true - kubectl -n $NS create configmap org0-config --from-file=config/org0 - kubectl -n $NS create configmap org1-config --from-file=config/org1 - kubectl -n $NS create configmap org2-config --from-file=config/org2 + kubectl -n $ORG0_NS create configmap org0-config --from-file=config/org0 + kubectl -n $ORG1_NS create configmap org1-config --from-file=config/org1 + kubectl -n $ORG2_NS create configmap org2-config --from-file=config/org2 pop_fn } @@ -61,8 +63,8 @@ function load_org_config() { function apply_k8s_builder_roles() { push_fn "Applying k8s chaincode builder roles" - apply_template kube/fabric-builder-role.yaml - apply_template kube/fabric-builder-rolebinding.yaml + apply_template kube/fabric-builder-role.yaml $ORG1_NS + apply_template kube/fabric-builder-rolebinding.yaml $ORG1_NS pop_fn } @@ -70,8 +72,8 @@ function apply_k8s_builder_roles() { function apply_k8s_builders() { push_fn "Installing k8s chaincode builders" - apply_template kube/org1/org1-install-k8s-builder.yaml - apply_template kube/org2/org2-install-k8s-builder.yaml + apply_template kube/org1/org1-install-k8s-builder.yaml $ORG1_NS + apply_template kube/org2/org2-install-k8s-builder.yaml $ORG1_NS pop_fn } \ No newline at end of file diff --git a/test-network-k8s/scripts/rest_sample.sh b/test-network-k8s/scripts/rest_sample.sh index c40c6b78..8da61efd 100755 --- a/test-network-k8s/scripts/rest_sample.sh +++ b/test-network-k8s/scripts/rest_sample.sh @@ -15,13 +15,16 @@ function json_ccp { local ORG=$1 local PP=$(one_line_pem $2) local CP=$(one_line_pem $3) + local NS=$4 sed -e "s/\${ORG}/$ORG/" \ -e "s#\${PEERPEM}#$PP#" \ -e "s#\${CAPEM}#$CP#" \ + -e "s#\${NS}#$NS#" \ scripts/ccp-template.json } function construct_rest_sample_configmap() { + local ns=$ORG1_NS push_fn "Constructing fabric-rest-sample connection profiles" ENROLLMENT_DIR=${TEMP_DIR}/enrollments @@ -32,11 +35,11 @@ function construct_rest_sample_configmap() { local peer_pem=$CHANNEL_MSP_DIR/peerOrganizations/org1/msp/tlscacerts/tlsca-signcert.pem local ca_pem=$CHANNEL_MSP_DIR/peerOrganizations/org1/msp/cacerts/ca-signcert.pem - echo "$(json_ccp 1 $peer_pem $ca_pem)" > build/fabric-rest-sample-config/HLF_CONNECTION_PROFILE_ORG1 + echo "$(json_ccp 1 $peer_pem $ca_pem $ORG1_NS)" > build/fabric-rest-sample-config/HLF_CONNECTION_PROFILE_ORG1 peer_pem=$CHANNEL_MSP_DIR/peerOrganizations/org2/msp/tlscacerts/tlsca-signcert.pem ca_pem=$CHANNEL_MSP_DIR/peerOrganizations/org2/msp/cacerts/ca-signcert.pem - echo "$(json_ccp 2 $peer_pem $ca_pem)" > build/fabric-rest-sample-config/HLF_CONNECTION_PROFILE_ORG2 + echo "$(json_ccp 2 $peer_pem $ca_pem $ORG2_NS)" > build/fabric-rest-sample-config/HLF_CONNECTION_PROFILE_ORG2 cp $ENROLLMENT_DIR/org1/users/org1admin/msp/signcerts/cert.pem $CONFIG_DIR/HLF_CERTIFICATE_ORG1 cp $ENROLLMENT_DIR/org2/users/org2admin/msp/signcerts/cert.pem $CONFIG_DIR/HLF_CERTIFICATE_ORG2 @@ -44,28 +47,29 @@ function construct_rest_sample_configmap() { cp $ENROLLMENT_DIR/org1/users/org1admin/msp/keystore/key.pem $CONFIG_DIR/HLF_PRIVATE_KEY_ORG1 cp $ENROLLMENT_DIR/org2/users/org2admin/msp/keystore/key.pem $CONFIG_DIR/HLF_PRIVATE_KEY_ORG2 - kubectl -n $NS delete configmap fabric-rest-sample-config || true - kubectl -n $NS create configmap fabric-rest-sample-config --from-file=$CONFIG_DIR + kubectl -n $ns delete configmap fabric-rest-sample-config || true + kubectl -n $ns create configmap fabric-rest-sample-config --from-file=$CONFIG_DIR pop_fn } function rollout_rest_sample() { + local ns=$ORG1_NS push_fn "Starting fabric-rest-sample" - kubectl -n $NS apply -f kube/fabric-rest-sample.yaml - kubectl -n $NS rollout status deploy/fabric-rest-sample + kubectl -n $ns apply -f kube/fabric-rest-sample.yaml + kubectl -n $ns rollout status deploy/fabric-rest-sample pop_fn } function launch_rest_sample() { - + local ns=$ORG1_NS construct_rest_sample_configmap - apply_template kube/fabric-rest-sample.yaml + apply_template kube/fabric-rest-sample.yaml $ns - kubectl -n $NS rollout status deploy/fabric-rest-sample + kubectl -n $ns rollout status deploy/fabric-rest-sample log "" log "The fabric-rest-sample has started." diff --git a/test-network-k8s/scripts/test_network.sh b/test-network-k8s/scripts/test_network.sh index b50b4b8a..018df67a 100755 --- a/test-network-k8s/scripts/test_network.sh +++ b/test-network-k8s/scripts/test_network.sh @@ -8,13 +8,13 @@ function launch_orderers() { push_fn "Launching orderers" - apply_template kube/org0/org0-orderer1.yaml - apply_template kube/org0/org0-orderer2.yaml - apply_template kube/org0/org0-orderer3.yaml + apply_template kube/org0/org0-orderer1.yaml $ORG0_NS + apply_template kube/org0/org0-orderer2.yaml $ORG0_NS + apply_template kube/org0/org0-orderer3.yaml $ORG0_NS - kubectl -n $NS rollout status deploy/org0-orderer1 - kubectl -n $NS rollout status deploy/org0-orderer2 - kubectl -n $NS rollout status deploy/org0-orderer3 + kubectl -n $ORG0_NS rollout status deploy/org0-orderer1 + kubectl -n $ORG0_NS rollout status deploy/org0-orderer2 + kubectl -n $ORG0_NS rollout status deploy/org0-orderer3 pop_fn } @@ -22,15 +22,15 @@ function launch_orderers() { function launch_peers() { push_fn "Launching peers" - apply_template kube/org1/org1-peer1.yaml - apply_template kube/org1/org1-peer2.yaml - apply_template kube/org2/org2-peer1.yaml - apply_template kube/org2/org2-peer2.yaml + apply_template kube/org1/org1-peer1.yaml $ORG1_NS + apply_template kube/org1/org1-peer2.yaml $ORG1_NS + apply_template kube/org2/org2-peer1.yaml $ORG2_NS + apply_template kube/org2/org2-peer2.yaml $ORG2_NS - kubectl -n $NS rollout status deploy/org1-peer1 - kubectl -n $NS rollout status deploy/org1-peer2 - kubectl -n $NS rollout status deploy/org2-peer1 - kubectl -n $NS rollout status deploy/org2-peer2 + kubectl -n $ORG1_NS rollout status deploy/org1-peer1 + kubectl -n $ORG1_NS rollout status deploy/org1-peer2 + kubectl -n $ORG2_NS rollout status deploy/org2-peer1 + kubectl -n $ORG2_NS rollout status deploy/org2-peer2 pop_fn } @@ -41,6 +41,7 @@ function create_node_local_MSP() { local org=$2 local node=$3 local csr_hosts=$4 + local ns=$5 local id_name=${org}-${node} local id_secret=${node_type}pw local ca_name=${org}-ca @@ -62,7 +63,7 @@ function create_node_local_MSP() { # Enroll the node admin user from within k8s. This will leave the certificates available on a volume share in the # cluster for access by the nodes when launching in a container. - cat < /dev/null - if [[ $? -ne 0 ]]; then - echo "No namespace $NS found - nothing to do." - return - fi - + for ns in $ORG0_NS $ORG1_NS $ORG2_NS; do + kubectl get namespace $ns > /dev/null + if [[ $? -ne 0 ]]; then + echo "No namespace $ns found - nothing to do." + return + fi + done set -e stop_services diff --git a/test-network-k8s/scripts/utils.sh b/test-network-k8s/scripts/utils.sh index c8f2cde6..4f1615c3 100644 --- a/test-network-k8s/scripts/utils.sh +++ b/test-network-k8s/scripts/utils.sh @@ -84,7 +84,7 @@ function apply_template() { echo "Applying template $1:" cat $1 | envsubst - cat $1 | envsubst | kubectl -n $NS apply -f - + cat $1 | envsubst | kubectl -n $2 apply -f - } # Set the calling context to refer the peer binary to the correct org / peer instance