2022/03/23 02:41:22 Starting Kubernetes Multicloud API 2022/03/23 02:41:22 Error Getting TLS Configuration. Starting without TLS... 10.233.72.78 - - [23/Mar/2022:03:42:00 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [23/Mar/2022:03:45:01 +0000] "POST /v1/rb/definition HTTP/1.1" 201 278 127.0.0.1 - - [23/Mar/2022:03:47:41 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 127.0.0.1 - - [23/Mar/2022:03:49:46 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.72.78 - - [23/Mar/2022:03:57:46 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.72.78 - - [23/Mar/2022:03:57:46 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7661 10.233.72.78 - - [23/Mar/2022:03:57:46 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7661 10.233.72.78 - - [23/Mar/2022:03:57:46 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.72.78 - - [23/Mar/2022:03:57:47 +0000] "GET /v1/rb/definition/pnf-cnf-rb-f31b740f-e40e-4fab-b05e-bdcaac499db3/v1 HTTP/1.1" 200 125 10.233.72.78 - - [23/Mar/2022:03:57:47 +0000] "POST /v1/rb/definition/pnf-cnf-rb-f31b740f-e40e-4fab-b05e-bdcaac499db3/v1/content HTTP/1.1" 200 0 10.233.72.78 - - [23/Mar/2022:03:57:47 +0000] "GET /v1/rb/definition/pnf-cnf-rb-f31b740f-e40e-4fab-b05e-bdcaac499db3/v1 HTTP/1.1" 200 138 10.233.72.78 - - [23/Mar/2022:03:57:47 +0000] "POST /v1/rb/definition/pnf-cnf-rb-f31b740f-e40e-4fab-b05e-bdcaac499db3/v1/profile HTTP/1.1" 201 308 10.233.72.78 - - [23/Mar/2022:03:57:47 +0000] "GET /v1/rb/definition/pnf-cnf-rb-f31b740f-e40e-4fab-b05e-bdcaac499db3/v1/profile/pnf-cnf-profile-b2bf57be-efe0-4ed6-8400-b9267075929e HTTP/1.1" 200 308 10.233.72.78 - - [23/Mar/2022:03:57:47 +0000] "POST /v1/rb/definition/pnf-cnf-rb-f31b740f-e40e-4fab-b05e-bdcaac499db3/v1/profile/pnf-cnf-profile-b2bf57be-efe0-4ed6-8400-b9267075929e/content HTTP/1.1" 200 0 2022/03/23 03:57:47 Error reading name cache from Database: Error finding master table: mongo: no documents in result 2022/03/23 03:57:47 Error Reading from DB: Reading cache from DB: Error finding master table: mongo: no documents in result 2022/03/23 03:57:47 Main rss info 2022/03/23 03:57:47 Path: /tmp/helm-tmpl-882777019/manifest-0 2022/03/23 03:57:47 Kind: Service 2022/03/23 03:57:47 Path: /tmp/helm-tmpl-882777019/manifest-1 2022/03/23 03:57:47 Kind: Service 2022/03/23 03:57:47 Path: /tmp/helm-tmpl-882777019/manifest-2 2022/03/23 03:57:47 Kind: Pod 2022/03/23 03:57:47 Path: /tmp/helm-tmpl-882777019/manifest-3 2022/03/23 03:57:47 Kind: Deployment 2022/03/23 03:57:47 Crd rss info 2022/03/23 03:57:47 Hook info {"filepath":"/tmp/helm-tmpl-882777019/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-23T03:57:47Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-23T03:57:47Z"} {"filepath":"/tmp/helm-tmpl-882777019/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-23T03:57:47Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-23T03:57:47Z"} {"filepath":"/tmp/helm-tmpl-882777019/manifest-2","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-23T03:57:47Z"} 2022/03/23 03:57:47 No plugin for kind Pod found. Using generic Plugin 2022/03/23 03:57:47 Error converting template to map {"gvk":{"Group":"","Version":"v1","Kind":"Pod"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-23T03:57:47Z"} {"filepath":"/tmp/helm-tmpl-882777019/manifest-3","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-23T03:57:47Z"} 2022/03/23 03:57:47 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-23T03:57:47Z"} 10.233.72.78 - - [23/Mar/2022:03:57:47 +0000] "POST /v1/instance HTTP/1.1" 201 729 10.233.72.78 - - [23/Mar/2022:04:00:14 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7661 127.0.0.1 - - [23/Mar/2022:04:00:27 +0000] "POST /v1/rb/definition HTTP/1.1" 201 265 127.0.0.1 - - [23/Mar/2022:04:00:27 +0000] "POST /v1/rb/definition/d553c831-aedf-41a2-a79a-576b292497d9/7beb6df5-1e11-4e42-b0d2-23a67f66e32d/content HTTP/1.1" 200 0 127.0.0.1 - - [23/Mar/2022:04:00:42 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 127.0.0.1 - - [23/Mar/2022:04:00:56 +0000] "POST /v1/rb/definition HTTP/1.1" 201 272 127.0.0.1 - - [23/Mar/2022:04:01:09 +0000] "POST /v1/rb/definition HTTP/1.1" 201 267 10.233.72.78 - - [23/Mar/2022:04:06:04 +0000] "GET /v1/rb/definition/d553c831-aedf-41a2-a79a-576b292497d9/7beb6df5-1e11-4e42-b0d2-23a67f66e32d HTTP/1.1" 200 270 10.233.72.78 - - [23/Mar/2022:04:06:05 +0000] "GET /v1/rb/definition/d553c831-aedf-41a2-a79a-576b292497d9/7beb6df5-1e11-4e42-b0d2-23a67f66e32d/profile/cnftest HTTP/1.1" 404 87 10.233.72.78 - - [23/Mar/2022:04:06:05 +0000] "POST /v1/rb/definition/d553c831-aedf-41a2-a79a-576b292497d9/7beb6df5-1e11-4e42-b0d2-23a67f66e32d/profile HTTP/1.1" 201 240 10.233.72.78 - - [23/Mar/2022:04:06:05 +0000] "GET /v1/rb/definition/d553c831-aedf-41a2-a79a-576b292497d9/7beb6df5-1e11-4e42-b0d2-23a67f66e32d/profile/cnftest HTTP/1.1" 200 240 10.233.72.78 - - [23/Mar/2022:04:06:05 +0000] "POST /v1/rb/definition/d553c831-aedf-41a2-a79a-576b292497d9/7beb6df5-1e11-4e42-b0d2-23a67f66e32d/profile/cnftest/content HTTP/1.1" 200 0 10.233.72.78 - - [23/Mar/2022:04:06:20 +0000] "GET /basicnf-owner/k8sregion/infra_workload?name=basic_cnf-0fda3932-ec71-4f7c-a7e7-26c38a983676 HTTP/1.1" 200 114 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"","workload_status":"GET_COMPLETE","workload_status_reason":{"stacks":[]}},"time":"2022-03-23T04:06:20Z"} {"Body":{},"Payload":{"generic-vnf-id":"8f9ec92e-dfcf-4717-a697-714f8dcf6e73","vf-module-id":"e05e704f-3d15-4c4e-a203-3283d03824d4","vf-module-model-invariant-id":"d553c831-aedf-41a2-a79a-576b292497d9","vf-module-model-version-id":"dc1ed460-e70d-4a7a-bcb8-60757cea33c9","vf-module-model-customization-id":"7beb6df5-1e11-4e42-b0d2-23a67f66e32d","oof_directives":{"attributes":null},"sdnc_directives":{"attributes":[{"attribute_name":"k8s-rb-profile-name","attribute_value":"cnftest"},{"attribute_name":"k8s-rb-profile-namespace","attribute_value":"k8s"},{"attribute_name":"availability_zone_0","attribute_value":"basicnf-availability-zone"},{"attribute_name":"dummy_name_0","attribute_value":"dummy_name"}]},"user_directives":{"attributes":null},"template_type":"","template_data":{"stack_name":"basic_cnf-0fda3932-ec71-4f7c-a7e7-26c38a983676"}},"level":"info","msg":"Broker API Payload","time":"2022-03-23T04:06:20Z"} {"level":"info","msg":"Instance API Payload","payload":{"rb-name":"d553c831-aedf-41a2-a79a-576b292497d9","rb-version":"7beb6df5-1e11-4e42-b0d2-23a67f66e32d","profile-name":"cnftest","release-name":"e05e704f-3d15-4c4e-a203-3283d03824d4","cloud-region":"k8sregion","labels":{"stack-name":"basic_cnf-0fda3932-ec71-4f7c-a7e7-26c38a983676"},"override-values":{"availability_zone_0":"basicnf-availability-zone","dummy_name_0":"dummy_name","k8s-rb-profile-name":"cnftest","k8s-rb-profile-namespace":"k8s"}},"time":"2022-03-23T04:06:20Z"} 2022/03/23 04:06:20 Main rss info 2022/03/23 04:06:20 Path: /tmp/helm-tmpl-124105927/manifest-0 2022/03/23 04:06:20 Kind: Service 2022/03/23 04:06:20 Path: /tmp/helm-tmpl-124105927/manifest-1 2022/03/23 04:06:20 Kind: Deployment 2022/03/23 04:06:20 Crd rss info 2022/03/23 04:06:20 Hook info {"level":"info","msg":"Creating Namespace","namespace":"k8s","time":"2022-03-23T04:06:20Z"} 2022/03/23 04:06:20 Namespace (k8s) created {"filepath":"/tmp/helm-tmpl-124105927/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-23T04:06:20Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-03-23T04:06:20Z"} {"filepath":"/tmp/helm-tmpl-124105927/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-23T04:06:20Z"} 2022/03/23 04:06:20 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-03-23T04:06:20Z"} {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"stupefied_cartwright","template_response":[{"GVK":{"Group":"","Version":"v1","Kind":"Service"},"Name":"cnftest-nginx"},{"GVK":{"Group":"apps","Version":"v1","Kind":"Deployment"},"Name":"cnftest-nginx"}],"workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-23T04:06:20Z"} 10.233.72.78 - - [23/Mar/2022:04:06:20 +0000] "POST /basicnf-owner/k8sregion/infra_workload HTTP/1.1" 201 309 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"stupefied_cartwright","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-23T04:06:21Z"} 10.233.72.78 - - [23/Mar/2022:04:06:21 +0000] "GET /basicnf-owner/k8sregion/infra_workload/stupefied_cartwright HTTP/1.1" 200 128 10.233.72.78 - - [23/Mar/2022:04:06:21 +0000] "POST /basicnf-owner/k8sregion/infra_workload/stupefied_cartwright HTTP/1.1" 405 0 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"stupefied_cartwright","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-03-23T04:06:51Z"} 10.233.72.78 - - [23/Mar/2022:04:06:51 +0000] "GET /basicnf-owner/k8sregion/infra_workload/stupefied_cartwright HTTP/1.1" 200 128 2022/03/23 04:06:52 [Config Cleanup] Instance stupefied_cartwright 2022/03/23 04:06:52 [scheduleResources]: START thread 2022/03/23 04:06:57 Stop config thread for stupefied_cartwright {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"warning","msg":"Deleting Resource","resource":"cnftest-nginx","time":"2022-03-23T04:06:57Z"} 2022/03/23 04:06:57 No plugin for kind Deployment found. Using generic Plugin 2022/03/23 04:06:57 [scheduleResources]: STOP thread {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"warning","msg":"Deleting Resource","resource":"cnftest-nginx","time":"2022-03-23T04:06:57Z"} 2022/03/23 04:06:57 Deleting service: cnftest-nginx 2022/03/23 04:06:57 Deleted Obj with ID ObjectID("623a9cbceafccfe38cd8550d") 2022/03/23 04:06:57 Deleted Obj with ID ObjectID("623a9cbc30d3389578d06a18") {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"stupefied_cartwright","workload_status":"DELETE_COMPLETE","workload_status_reason":null},"time":"2022-03-23T04:06:57Z"} 10.233.72.78 - - [23/Mar/2022:04:06:52 +0000] "DELETE /basicnf-owner/k8sregion/infra_workload/stupefied_cartwright HTTP/1.1" 202 128 10.233.72.78 - - [23/Mar/2022:04:07:07 +0000] "GET /v1/rb/definition/d553c831-aedf-41a2-a79a-576b292497d9/7beb6df5-1e11-4e42-b0d2-23a67f66e32d HTTP/1.1" 200 270 10.233.72.78 - - [23/Mar/2022:04:07:07 +0000] "GET /v1/rb/definition/d553c831-aedf-41a2-a79a-576b292497d9/7beb6df5-1e11-4e42-b0d2-23a67f66e32d/profile/cnftest HTTP/1.1" 200 240 2022/03/23 04:07:07 Deleted Obj with ID ObjectID("623a9cadeafccfe38cd8550b") 2022/03/23 04:07:07 Deleted Obj with ID ObjectID("623a9cadeafccfe38cd8550c") 10.233.72.78 - - [23/Mar/2022:04:07:07 +0000] "DELETE /v1/rb/definition/d553c831-aedf-41a2-a79a-576b292497d9/7beb6df5-1e11-4e42-b0d2-23a67f66e32d/profile/cnftest HTTP/1.1" 204 0 2022/03/23 04:07:07 Deleted Obj with ID ObjectID("623a9cac30d3389578d069ed") 10.233.72.78 - - [23/Mar/2022:04:07:38 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7661 2022/03/23 04:07:38 Deleted Obj with ID ObjectID("623a9abaeafccfe38cd854f3") 10.233.72.78 - - [23/Mar/2022:04:07:38 +0000] "DELETE /v1/connectivity-info/k8sregion HTTP/1.1" 204 0 2022/03/23 04:07:38 Deleted Obj with ID ObjectID("623a9aba30d3389578d06831") {"error":"Getting CloudRegion Information: Get kubeconfig file: Downloading kubeconfig: Getting Connection info: Get Connection: Error finding master table: mongo: no documents in result","level":"error","msg":"Error Deleting Instance","time":"2022-03-23T04:11:41Z"} 10.233.72.78 - - [23/Mar/2022:04:11:41 +0000] "DELETE /v1/instance/gallant_wescoff HTTP/1.1" 500 177 10.233.72.78 - - [23/Mar/2022:04:14:14 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.72.78 - - [23/Mar/2022:04:14:14 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7661 10.233.72.78 - - [23/Mar/2022:04:14:14 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7661 10.233.72.78 - - [23/Mar/2022:04:14:14 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.72.78 - - [23/Mar/2022:04:14:14 +0000] "GET /v1/rb/definition/pnf-cnf-rb-d1c5fbbc-a2ef-4a30-b806-554f58b9d741/v1 HTTP/1.1" 200 125 10.233.72.78 - - [23/Mar/2022:04:14:14 +0000] "POST /v1/rb/definition/pnf-cnf-rb-d1c5fbbc-a2ef-4a30-b806-554f58b9d741/v1/content HTTP/1.1" 200 0 10.233.72.78 - - [23/Mar/2022:04:14:14 +0000] "GET /v1/rb/definition/pnf-cnf-rb-d1c5fbbc-a2ef-4a30-b806-554f58b9d741/v1 HTTP/1.1" 200 138 10.233.72.78 - - [23/Mar/2022:04:14:14 +0000] "POST /v1/rb/definition/pnf-cnf-rb-d1c5fbbc-a2ef-4a30-b806-554f58b9d741/v1/profile HTTP/1.1" 201 308 10.233.72.78 - - [23/Mar/2022:04:14:14 +0000] "GET /v1/rb/definition/pnf-cnf-rb-d1c5fbbc-a2ef-4a30-b806-554f58b9d741/v1/profile/pnf-cnf-profile-7e0bb5a1-e890-48e6-9b57-f8261ce18c27 HTTP/1.1" 200 308 10.233.72.78 - - [23/Mar/2022:04:14:14 +0000] "POST /v1/rb/definition/pnf-cnf-rb-d1c5fbbc-a2ef-4a30-b806-554f58b9d741/v1/profile/pnf-cnf-profile-7e0bb5a1-e890-48e6-9b57-f8261ce18c27/content HTTP/1.1" 200 0 2022/03/23 04:14:14 Main rss info 2022/03/23 04:14:14 Path: /tmp/helm-tmpl-277647957/manifest-0 2022/03/23 04:14:14 Kind: Service 2022/03/23 04:14:14 Path: /tmp/helm-tmpl-277647957/manifest-1 2022/03/23 04:14:14 Kind: Service 2022/03/23 04:14:14 Path: /tmp/helm-tmpl-277647957/manifest-2 2022/03/23 04:14:14 Kind: Pod 2022/03/23 04:14:14 Path: /tmp/helm-tmpl-277647957/manifest-3 2022/03/23 04:14:14 Kind: Deployment 2022/03/23 04:14:14 Crd rss info 2022/03/23 04:14:14 Hook info {"filepath":"/tmp/helm-tmpl-277647957/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-23T04:14:14Z"} {"error":"Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","filepath":"/tmp/helm-tmpl-277647957/manifest-0","gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"error","msg":"Error Creating Resource","time":"2022-03-23T04:14:14Z"} 2022/03/23 04:14:14 Instance: blissful_fermat, Main rss are failed, skip post-install and remove instance in DB 2022/03/23 04:14:14 Deleted Obj with ID ObjectID("623a9e96eafccfe38cd85516") 2022/03/23 04:14:14 Deleted Obj with ID ObjectID("623a9e9630d3389578d06b9f") {"error":"Create Kubernetes Resources: Error creating kind: /v1, Kind=Service: Error in plugin Service plugin: Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","level":"error","msg":"Error Creating Resource","resource":{"rb-name":"pnf-cnf-rb-d1c5fbbc-a2ef-4a30-b806-554f58b9d741","rb-version":"v1","profile-name":"pnf-cnf-profile-7e0bb5a1-e890-48e6-9b57-f8261ce18c27","release-name":"","cloud-region":"k8sregion","labels":{},"override-values":{}},"time":"2022-03-23T04:14:14Z"} 10.233.72.78 - - [23/Mar/2022:04:14:14 +0000] "POST /v1/instance HTTP/1.1" 500 180