2022/03/14 02:42:43 Starting Kubernetes Multicloud API 2022/03/14 02:42:43 Error Getting TLS Configuration. Starting without TLS... 10.233.69.167 - - [14/Mar/2022:03:47:34 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [14/Mar/2022:03:50:21 +0000] "POST /v1/rb/definition HTTP/1.1" 201 278 127.0.0.1 - - [14/Mar/2022:03:53:07 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 127.0.0.1 - - [14/Mar/2022:03:55:40 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.69.167 - - [14/Mar/2022:04:04:31 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.69.167 - - [14/Mar/2022:04:04:31 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7669 10.233.69.167 - - [14/Mar/2022:04:04:31 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7669 10.233.69.167 - - [14/Mar/2022:04:04:31 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.69.167 - - [14/Mar/2022:04:04:31 +0000] "GET /v1/rb/definition/pnf-cnf-rb-d4c0cde0-fe58-41b9-8582-7a674add7e50/v1 HTTP/1.1" 200 125 10.233.69.167 - - [14/Mar/2022:04:04:31 +0000] "POST /v1/rb/definition/pnf-cnf-rb-d4c0cde0-fe58-41b9-8582-7a674add7e50/v1/content HTTP/1.1" 200 0 10.233.69.167 - - [14/Mar/2022:04:04:31 +0000] "GET /v1/rb/definition/pnf-cnf-rb-d4c0cde0-fe58-41b9-8582-7a674add7e50/v1 HTTP/1.1" 200 138 10.233.69.167 - - [14/Mar/2022:04:04:31 +0000] "POST /v1/rb/definition/pnf-cnf-rb-d4c0cde0-fe58-41b9-8582-7a674add7e50/v1/profile HTTP/1.1" 201 308 10.233.69.167 - - [14/Mar/2022:04:04:31 +0000] "GET /v1/rb/definition/pnf-cnf-rb-d4c0cde0-fe58-41b9-8582-7a674add7e50/v1/profile/pnf-cnf-profile-c1406383-9ecb-4a41-b141-262b310d4617 HTTP/1.1" 200 308 10.233.69.167 - - [14/Mar/2022:04:04:31 +0000] "POST /v1/rb/definition/pnf-cnf-rb-d4c0cde0-fe58-41b9-8582-7a674add7e50/v1/profile/pnf-cnf-profile-c1406383-9ecb-4a41-b141-262b310d4617/content HTTP/1.1" 200 0 2022/03/14 04:04:31 Error reading name cache from Database: Error finding master table: mongo: no documents in result 2022/03/14 04:04:31 Error Reading from DB: Reading cache from DB: Error finding master table: mongo: no documents in result 2022/03/14 04:04:31 Main rss info 2022/03/14 04:04:31 Path: /tmp/helm-tmpl-524517255/manifest-0 2022/03/14 04:04:31 Kind: Service 2022/03/14 04:04:31 Path: /tmp/helm-tmpl-524517255/manifest-1 2022/03/14 04:04:31 Kind: Service 2022/03/14 04:04:31 Path: /tmp/helm-tmpl-524517255/manifest-2 2022/03/14 04:04:31 Kind: Pod 2022/03/14 04:04:31 Path: /tmp/helm-tmpl-524517255/manifest-3 2022/03/14 04:04:31 Kind: Deployment 2022/03/14 04:04:31 Crd rss info 2022/03/14 04:04:31 Hook info {"filepath":"/tmp/helm-tmpl-524517255/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-14T04:04:31Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-14T04:04:31Z"} {"filepath":"/tmp/helm-tmpl-524517255/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-14T04:04:31Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-14T04:04:31Z"} {"filepath":"/tmp/helm-tmpl-524517255/manifest-2","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-14T04:04:31Z"} 2022/03/14 04:04:31 No plugin for kind Pod found. Using generic Plugin 2022/03/14 04:04:31 Error converting template to map {"gvk":{"Group":"","Version":"v1","Kind":"Pod"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-14T04:04:31Z"} {"filepath":"/tmp/helm-tmpl-524517255/manifest-3","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-14T04:04:31Z"} 2022/03/14 04:04:31 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-14T04:04:31Z"} 10.233.69.167 - - [14/Mar/2022:04:04:31 +0000] "POST /v1/instance HTTP/1.1" 201 730 10.233.69.167 - - [14/Mar/2022:04:07:18 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7669 127.0.0.1 - - [14/Mar/2022:04:07:27 +0000] "POST /v1/rb/definition HTTP/1.1" 201 265 127.0.0.1 - - [14/Mar/2022:04:07:27 +0000] "POST /v1/rb/definition/2f0013b3-5586-49c0-9739-04636fc994e2/974c62fa-5928-4c60-acc1-1c0d4b70a0b0/content HTTP/1.1" 200 0 127.0.0.1 - - [14/Mar/2022:04:08:21 +0000] "POST /v1/rb/definition HTTP/1.1" 201 272 127.0.0.1 - - [14/Mar/2022:04:08:34 +0000] "POST /v1/rb/definition HTTP/1.1" 201 267 10.233.69.167 - - [14/Mar/2022:04:13:08 +0000] "GET /v1/rb/definition/2f0013b3-5586-49c0-9739-04636fc994e2/974c62fa-5928-4c60-acc1-1c0d4b70a0b0 HTTP/1.1" 200 270 10.233.69.167 - - [14/Mar/2022:04:13:08 +0000] "GET /v1/rb/definition/2f0013b3-5586-49c0-9739-04636fc994e2/974c62fa-5928-4c60-acc1-1c0d4b70a0b0/profile/cnftest HTTP/1.1" 404 87 10.233.69.167 - - [14/Mar/2022:04:13:08 +0000] "POST /v1/rb/definition/2f0013b3-5586-49c0-9739-04636fc994e2/974c62fa-5928-4c60-acc1-1c0d4b70a0b0/profile HTTP/1.1" 201 240 10.233.69.167 - - [14/Mar/2022:04:13:08 +0000] "GET /v1/rb/definition/2f0013b3-5586-49c0-9739-04636fc994e2/974c62fa-5928-4c60-acc1-1c0d4b70a0b0/profile/cnftest HTTP/1.1" 200 240 10.233.69.167 - - [14/Mar/2022:04:13:08 +0000] "POST /v1/rb/definition/2f0013b3-5586-49c0-9739-04636fc994e2/974c62fa-5928-4c60-acc1-1c0d4b70a0b0/profile/cnftest/content HTTP/1.1" 200 0 10.233.69.167 - - [14/Mar/2022:04:13:27 +0000] "GET /basicnf-owner/k8sregion/infra_workload?name=basic_cnf-0147d206-2491-4751-a976-74c7b8dd541f HTTP/1.1" 200 114 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"","workload_status":"GET_COMPLETE","workload_status_reason":{"stacks":[]}},"time":"2022-03-14T04:13:27Z"} {"Body":{},"Payload":{"generic-vnf-id":"a33012a7-baa3-4cc6-a209-b1d806cece39","vf-module-id":"f6f96e8b-9258-4778-b274-69d72d5707fc","vf-module-model-invariant-id":"2f0013b3-5586-49c0-9739-04636fc994e2","vf-module-model-version-id":"57f4b1a1-ccb5-4273-8855-0135d8263854","vf-module-model-customization-id":"974c62fa-5928-4c60-acc1-1c0d4b70a0b0","oof_directives":{"attributes":null},"sdnc_directives":{"attributes":[{"attribute_name":"k8s-rb-profile-name","attribute_value":"cnftest"},{"attribute_name":"k8s-rb-profile-namespace","attribute_value":"k8s"},{"attribute_name":"availability_zone_0","attribute_value":"basicnf-availability-zone"},{"attribute_name":"dummy_name_0","attribute_value":"dummy_name"}]},"user_directives":{"attributes":null},"template_type":"","template_data":{"stack_name":"basic_cnf-0147d206-2491-4751-a976-74c7b8dd541f"}},"level":"info","msg":"Broker API Payload","time":"2022-03-14T04:13:27Z"} {"level":"info","msg":"Instance API Payload","payload":{"rb-name":"2f0013b3-5586-49c0-9739-04636fc994e2","rb-version":"974c62fa-5928-4c60-acc1-1c0d4b70a0b0","profile-name":"cnftest","release-name":"f6f96e8b-9258-4778-b274-69d72d5707fc","cloud-region":"k8sregion","labels":{"stack-name":"basic_cnf-0147d206-2491-4751-a976-74c7b8dd541f"},"override-values":{"availability_zone_0":"basicnf-availability-zone","dummy_name_0":"dummy_name","k8s-rb-profile-name":"cnftest","k8s-rb-profile-namespace":"k8s"}},"time":"2022-03-14T04:13:27Z"} 2022/03/14 04:13:27 Main rss info 2022/03/14 04:13:27 Path: /tmp/helm-tmpl-004325075/manifest-0 2022/03/14 04:13:27 Kind: Service 2022/03/14 04:13:27 Path: /tmp/helm-tmpl-004325075/manifest-1 2022/03/14 04:13:27 Kind: Deployment 2022/03/14 04:13:27 Crd rss info 2022/03/14 04:13:27 Hook info {"level":"info","msg":"Creating Namespace","namespace":"k8s","time":"2022-03-14T04:13:27Z"} 2022/03/14 04:13:27 Namespace (k8s) created {"filepath":"/tmp/helm-tmpl-004325075/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-14T04:13:27Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-03-14T04:13:27Z"} {"filepath":"/tmp/helm-tmpl-004325075/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-14T04:13:27Z"} 2022/03/14 04:13:27 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-03-14T04:13:27Z"} 10.233.69.167 - - [14/Mar/2022:04:13:27 +0000] "POST /basicnf-owner/k8sregion/infra_workload HTTP/1.1" 201 301 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"brave_merkle","template_response":[{"GVK":{"Group":"","Version":"v1","Kind":"Service"},"Name":"cnftest-nginx"},{"GVK":{"Group":"apps","Version":"v1","Kind":"Deployment"},"Name":"cnftest-nginx"}],"workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-14T04:13:27Z"} {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"brave_merkle","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-14T04:13:27Z"} 10.233.69.167 - - [14/Mar/2022:04:13:27 +0000] "GET /basicnf-owner/k8sregion/infra_workload/brave_merkle HTTP/1.1" 200 120 10.233.69.167 - - [14/Mar/2022:04:13:27 +0000] "POST /basicnf-owner/k8sregion/infra_workload/brave_merkle HTTP/1.1" 405 0 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"brave_merkle","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-14T04:14:07Z"} 10.233.69.167 - - [14/Mar/2022:04:14:07 +0000] "GET /basicnf-owner/k8sregion/infra_workload/brave_merkle HTTP/1.1" 200 120 2022/03/14 04:14:07 [Config Cleanup] Instance brave_merkle 2022/03/14 04:14:07 [scheduleResources]: START thread 2022/03/14 04:14:12 Stop config thread for brave_merkle {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"warning","msg":"Deleting Resource","resource":"cnftest-nginx","time":"2022-03-14T04:14:12Z"} 2022/03/14 04:14:12 No plugin for kind Deployment found. Using generic Plugin 2022/03/14 04:14:12 [scheduleResources]: STOP thread {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"warning","msg":"Deleting Resource","resource":"cnftest-nginx","time":"2022-03-14T04:14:12Z"} 2022/03/14 04:14:12 Deleting service: cnftest-nginx 2022/03/14 04:14:12 Deleted Obj with ID ObjectID("622ec0e7403f99b373ab7f9d") 2022/03/14 04:14:12 Deleted Obj with ID ObjectID("622ec0e6fde46573a595e7e0") {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"brave_merkle","workload_status":"DELETE_COMPLETE","workload_status_reason":null},"time":"2022-03-14T04:14:12Z"} 10.233.69.167 - - [14/Mar/2022:04:14:07 +0000] "DELETE /basicnf-owner/k8sregion/infra_workload/brave_merkle HTTP/1.1" 202 120 10.233.69.167 - - [14/Mar/2022:04:14:22 +0000] "GET /v1/rb/definition/2f0013b3-5586-49c0-9739-04636fc994e2/974c62fa-5928-4c60-acc1-1c0d4b70a0b0 HTTP/1.1" 200 270 10.233.69.167 - - [14/Mar/2022:04:14:22 +0000] "GET /v1/rb/definition/2f0013b3-5586-49c0-9739-04636fc994e2/974c62fa-5928-4c60-acc1-1c0d4b70a0b0/profile/cnftest HTTP/1.1" 200 240 2022/03/14 04:14:22 Deleted Obj with ID ObjectID("622ec0d4403f99b373ab7f9b") 2022/03/14 04:14:22 Deleted Obj with ID ObjectID("622ec0d4403f99b373ab7f9c") 2022/03/14 04:14:22 Deleted Obj with ID ObjectID("622ec0d4fde46573a595e7b8") 10.233.69.167 - - [14/Mar/2022:04:14:22 +0000] "DELETE /v1/rb/definition/2f0013b3-5586-49c0-9739-04636fc994e2/974c62fa-5928-4c60-acc1-1c0d4b70a0b0/profile/cnftest HTTP/1.1" 204 0 10.233.69.167 - - [14/Mar/2022:04:14:53 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7669 2022/03/14 04:14:53 Deleted Obj with ID ObjectID("622ebecf403f99b373ab7f86") 10.233.69.167 - - [14/Mar/2022:04:14:53 +0000] "DELETE /v1/connectivity-info/k8sregion HTTP/1.1" 204 0 2022/03/14 04:14:53 Deleted Obj with ID ObjectID("622ebecefde46573a595e600") 10.233.69.167 - - [14/Mar/2022:04:16:50 +0000] "DELETE /v1/instance/cranky_mendeleev HTTP/1.1" 500 177 {"error":"Getting CloudRegion Information: Get kubeconfig file: Downloading kubeconfig: Getting Connection info: Get Connection: Error finding master table: mongo: no documents in result","level":"error","msg":"Error Deleting Instance","time":"2022-03-14T04:16:50Z"} 10.233.69.167 - - [14/Mar/2022:04:20:21 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.69.167 - - [14/Mar/2022:04:20:21 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7669 10.233.69.167 - - [14/Mar/2022:04:20:21 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7669 10.233.69.167 - - [14/Mar/2022:04:20:21 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.69.167 - - [14/Mar/2022:04:20:21 +0000] "GET /v1/rb/definition/pnf-cnf-rb-a0260fb4-116a-4da4-9f42-870870d7fe60/v1 HTTP/1.1" 200 125 10.233.69.167 - - [14/Mar/2022:04:20:21 +0000] "POST /v1/rb/definition/pnf-cnf-rb-a0260fb4-116a-4da4-9f42-870870d7fe60/v1/content HTTP/1.1" 200 0 10.233.69.167 - - [14/Mar/2022:04:20:21 +0000] "GET /v1/rb/definition/pnf-cnf-rb-a0260fb4-116a-4da4-9f42-870870d7fe60/v1 HTTP/1.1" 200 138 10.233.69.167 - - [14/Mar/2022:04:20:21 +0000] "POST /v1/rb/definition/pnf-cnf-rb-a0260fb4-116a-4da4-9f42-870870d7fe60/v1/profile HTTP/1.1" 201 308 10.233.69.167 - - [14/Mar/2022:04:20:21 +0000] "GET /v1/rb/definition/pnf-cnf-rb-a0260fb4-116a-4da4-9f42-870870d7fe60/v1/profile/pnf-cnf-profile-f37f1518-e799-4ecf-89ea-35a46e90b234 HTTP/1.1" 200 308 10.233.69.167 - - [14/Mar/2022:04:20:21 +0000] "POST /v1/rb/definition/pnf-cnf-rb-a0260fb4-116a-4da4-9f42-870870d7fe60/v1/profile/pnf-cnf-profile-f37f1518-e799-4ecf-89ea-35a46e90b234/content HTTP/1.1" 200 0 2022/03/14 04:20:21 Main rss info 2022/03/14 04:20:21 Path: /tmp/helm-tmpl-602547457/manifest-0 2022/03/14 04:20:21 Kind: Service 2022/03/14 04:20:21 Path: /tmp/helm-tmpl-602547457/manifest-1 2022/03/14 04:20:21 Kind: Service 2022/03/14 04:20:21 Path: /tmp/helm-tmpl-602547457/manifest-2 2022/03/14 04:20:21 Kind: Pod 2022/03/14 04:20:21 Path: /tmp/helm-tmpl-602547457/manifest-3 2022/03/14 04:20:21 Kind: Deployment 2022/03/14 04:20:21 Crd rss info 2022/03/14 04:20:21 Hook info {"filepath":"/tmp/helm-tmpl-602547457/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-14T04:20:21Z"} {"error":"Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","filepath":"/tmp/helm-tmpl-602547457/manifest-0","gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"error","msg":"Error Creating Resource","time":"2022-03-14T04:20:21Z"} 2022/03/14 04:20:21 Instance: eloquent_swanson, Main rss are failed, skip post-install and remove instance in DB 2022/03/14 04:20:21 Deleted Obj with ID ObjectID("622ec285403f99b373ab7fa6") 2022/03/14 04:20:21 Deleted Obj with ID ObjectID("622ec284fde46573a595e948") {"error":"Create Kubernetes Resources: Error creating kind: /v1, Kind=Service: Error in plugin Service plugin: Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","level":"error","msg":"Error Creating Resource","resource":{"rb-name":"pnf-cnf-rb-a0260fb4-116a-4da4-9f42-870870d7fe60","rb-version":"v1","profile-name":"pnf-cnf-profile-f37f1518-e799-4ecf-89ea-35a46e90b234","release-name":"","cloud-region":"k8sregion","labels":{},"override-values":{}},"time":"2022-03-14T04:20:21Z"} 10.233.69.167 - - [14/Mar/2022:04:20:21 +0000] "POST /v1/instance HTTP/1.1" 500 180