2021/10/27 01:15:04 Etcd Client Initialization failed with error: Error creating etcd client: dial tcp: lookup multicloud-k8s-etcd on 169.254.25.10:53: no such host 2021/10/27 01:15:05 Starting Kubernetes Multicloud API 2021/10/27 01:15:05 Error Getting TLS Configuration. Starting without TLS... 10.233.78.15 - - [27/Oct/2021:05:30:27 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.78.15 - - [27/Oct/2021:05:31:31 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.78.15 - - [27/Oct/2021:05:32:30 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [27/Oct/2021:05:33:25 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.78.15 - - [27/Oct/2021:05:33:31 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [27/Oct/2021:05:33:57 +0000] "POST /v1/rb/definition HTTP/1.1" 201 278 10.233.78.15 - - [27/Oct/2021:05:34:20 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.78.15 - - [27/Oct/2021:05:35:16 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.78.15 - - [27/Oct/2021:05:36:21 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.78.15 - - [27/Oct/2021:05:37:42 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.78.15 - - [27/Oct/2021:05:39:03 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.78.15 - - [27/Oct/2021:05:41:43 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [27/Oct/2021:05:49:18 +0000] "POST /v1/rb/definition HTTP/1.1" 201 267 127.0.0.1 - - [27/Oct/2021:05:49:39 +0000] "POST /v1/rb/definition HTTP/1.1" 201 272 10.233.78.15 - - [27/Oct/2021:05:51:16 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.78.15 - - [27/Oct/2021:05:51:16 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7569 10.233.78.15 - - [27/Oct/2021:05:51:16 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7569 10.233.78.15 - - [27/Oct/2021:05:51:16 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.78.15 - - [27/Oct/2021:05:51:16 +0000] "GET /v1/rb/definition/pnf-cnf-rb-81b9dcdc-f601-4c80-a000-e3e7c339b2ff/v1 HTTP/1.1" 200 125 10.233.78.15 - - [27/Oct/2021:05:51:16 +0000] "POST /v1/rb/definition/pnf-cnf-rb-81b9dcdc-f601-4c80-a000-e3e7c339b2ff/v1/content HTTP/1.1" 200 0 10.233.78.15 - - [27/Oct/2021:05:51:16 +0000] "GET /v1/rb/definition/pnf-cnf-rb-81b9dcdc-f601-4c80-a000-e3e7c339b2ff/v1 HTTP/1.1" 200 138 10.233.78.15 - - [27/Oct/2021:05:51:16 +0000] "POST /v1/rb/definition/pnf-cnf-rb-81b9dcdc-f601-4c80-a000-e3e7c339b2ff/v1/profile HTTP/1.1" 201 280 10.233.78.15 - - [27/Oct/2021:05:51:16 +0000] "GET /v1/rb/definition/pnf-cnf-rb-81b9dcdc-f601-4c80-a000-e3e7c339b2ff/v1/profile/pnf-cnf-profile-4249cff3-e726-4374-9f1d-67b58b993965 HTTP/1.1" 200 280 10.233.78.15 - - [27/Oct/2021:05:51:16 +0000] "POST /v1/rb/definition/pnf-cnf-rb-81b9dcdc-f601-4c80-a000-e3e7c339b2ff/v1/profile/pnf-cnf-profile-4249cff3-e726-4374-9f1d-67b58b993965/content HTTP/1.1" 200 0 2021/10/27 05:51:17 Error reading name cache from Database: Error finding master table: mongo: no documents in result 2021/10/27 05:51:17 Error Reading from DB: Reading cache from DB: Error finding master table: mongo: no documents in result 2021/10/27 05:51:17 Main rss info 2021/10/27 05:51:17 Path: /tmp/helm-tmpl-558020443/manifest-0 2021/10/27 05:51:17 Kind: Service 2021/10/27 05:51:17 Path: /tmp/helm-tmpl-558020443/manifest-1 2021/10/27 05:51:17 Kind: Service 2021/10/27 05:51:17 Path: /tmp/helm-tmpl-558020443/manifest-2 2021/10/27 05:51:17 Kind: Pod 2021/10/27 05:51:17 Path: /tmp/helm-tmpl-558020443/manifest-3 2021/10/27 05:51:17 Kind: Deployment 2021/10/27 05:51:17 Crd rss info 2021/10/27 05:51:17 Hook info {"filepath":"/tmp/helm-tmpl-558020443/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2021-10-27T05:51:17Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2021-10-27T05:51:17Z"} {"filepath":"/tmp/helm-tmpl-558020443/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2021-10-27T05:51:17Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2021-10-27T05:51:17Z"} {"filepath":"/tmp/helm-tmpl-558020443/manifest-2","level":"info","msg":"Processing Kubernetes Resource","time":"2021-10-27T05:51:17Z"} 2021/10/27 05:51:17 No plugin for kind Pod found. Using generic Plugin 2021/10/27 05:51:18 Error converting template to map {"gvk":{"Group":"","Version":"v1","Kind":"Pod"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2021-10-27T05:51:18Z"} {"filepath":"/tmp/helm-tmpl-558020443/manifest-3","level":"info","msg":"Processing Kubernetes Resource","time":"2021-10-27T05:51:18Z"} 2021/10/27 05:51:18 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2021-10-27T05:51:18Z"} 10.233.78.15 - - [27/Oct/2021:05:51:17 +0000] "POST /v1/instance HTTP/1.1" 201 731 10.233.78.15 - - [27/Oct/2021:05:53:00 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7569 127.0.0.1 - - [27/Oct/2021:05:53:22 +0000] "POST /v1/rb/definition HTTP/1.1" 201 265 127.0.0.1 - - [27/Oct/2021:05:53:22 +0000] "POST /v1/rb/definition/dff8b1c9-f3e4-447f-947b-fd88d69184a4/d98dbdf5-e0d0-4982-8948-8a0aaa0f1901/content HTTP/1.1" 200 0 127.0.0.1 - - [27/Oct/2021:05:53:59 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.78.15 - - [27/Oct/2021:05:57:43 +0000] "GET /v1/rb/definition/dff8b1c9-f3e4-447f-947b-fd88d69184a4/d98dbdf5-e0d0-4982-8948-8a0aaa0f1901 HTTP/1.1" 200 270 10.233.78.15 - - [27/Oct/2021:05:57:43 +0000] "GET /v1/rb/definition/dff8b1c9-f3e4-447f-947b-fd88d69184a4/d98dbdf5-e0d0-4982-8948-8a0aaa0f1901/profile/cnftest HTTP/1.1" 404 87 10.233.78.15 - - [27/Oct/2021:05:57:43 +0000] "POST /v1/rb/definition/dff8b1c9-f3e4-447f-947b-fd88d69184a4/d98dbdf5-e0d0-4982-8948-8a0aaa0f1901/profile HTTP/1.1" 201 212 10.233.78.15 - - [27/Oct/2021:05:57:43 +0000] "GET /v1/rb/definition/dff8b1c9-f3e4-447f-947b-fd88d69184a4/d98dbdf5-e0d0-4982-8948-8a0aaa0f1901/profile/cnftest HTTP/1.1" 200 212 10.233.78.15 - - [27/Oct/2021:05:57:44 +0000] "POST /v1/rb/definition/dff8b1c9-f3e4-447f-947b-fd88d69184a4/d98dbdf5-e0d0-4982-8948-8a0aaa0f1901/profile/cnftest/content HTTP/1.1" 200 0 10.233.78.15 - - [27/Oct/2021:05:57:57 +0000] "GET /basicnf-owner/k8sregion/infra_workload?name=basic_cnf-e96a9b4b-4fce-48da-8c0d-fe327e9e8b74 HTTP/1.1" 200 114 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"","workload_status":"GET_COMPLETE","workload_status_reason":{"stacks":[]}},"time":"2021-10-27T05:57:57Z"} {"Body":{},"Payload":{"generic-vnf-id":"6197a54b-a04d-4e4d-87be-b7623b507478","vf-module-id":"9f886b61-86b2-4749-bab7-3eeaf5ac1f99","vf-module-model-invariant-id":"dff8b1c9-f3e4-447f-947b-fd88d69184a4","vf-module-model-version-id":"bbe774a8-1451-4e3d-b270-b1f320201740","vf-module-model-customization-id":"d98dbdf5-e0d0-4982-8948-8a0aaa0f1901","oof_directives":{"attributes":null},"sdnc_directives":{"attributes":[{"attribute_name":"k8s-rb-profile-name","attribute_value":"cnftest"},{"attribute_name":"k8s-rb-profile-namespace","attribute_value":"k8s"},{"attribute_name":"availability_zone_0","attribute_value":"basicnf-availability-zone"},{"attribute_name":"dummy_name_0","attribute_value":"dummy_name"}]},"user_directives":{"attributes":null},"template_type":"","template_data":{"stack_name":"basic_cnf-e96a9b4b-4fce-48da-8c0d-fe327e9e8b74"}},"level":"info","msg":"Broker API Payload","time":"2021-10-27T05:57:58Z"} {"level":"info","msg":"Instance API Payload","payload":{"rb-name":"dff8b1c9-f3e4-447f-947b-fd88d69184a4","rb-version":"d98dbdf5-e0d0-4982-8948-8a0aaa0f1901","profile-name":"cnftest","release-name":"9f886b61-86b2-4749-bab7-3eeaf5ac1f99","cloud-region":"k8sregion","labels":{"stack-name":"basic_cnf-e96a9b4b-4fce-48da-8c0d-fe327e9e8b74"},"override-values":{"availability_zone_0":"basicnf-availability-zone","dummy_name_0":"dummy_name","k8s-rb-profile-name":"cnftest","k8s-rb-profile-namespace":"k8s"}},"time":"2021-10-27T05:57:58Z"} 2021/10/27 05:57:58 Main rss info 2021/10/27 05:57:58 Path: /tmp/helm-tmpl-826844782/manifest-0 2021/10/27 05:57:58 Kind: Service 2021/10/27 05:57:58 Path: /tmp/helm-tmpl-826844782/manifest-1 2021/10/27 05:57:58 Kind: Deployment 2021/10/27 05:57:58 Crd rss info 2021/10/27 05:57:58 Hook info {"level":"info","msg":"Creating Namespace","namespace":"k8s","time":"2021-10-27T05:57:58Z"} 2021/10/27 05:57:58 Namespace (k8s) created {"filepath":"/tmp/helm-tmpl-826844782/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2021-10-27T05:57:58Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2021-10-27T05:57:58Z"} {"filepath":"/tmp/helm-tmpl-826844782/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2021-10-27T05:57:58Z"} 2021/10/27 05:57:58 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2021-10-27T05:57:58Z"} {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"hardcore_golick","template_response":[{"GVK":{"Group":"","Version":"v1","Kind":"Service"},"Name":"cnftest-nginx"},{"GVK":{"Group":"apps","Version":"v1","Kind":"Deployment"},"Name":"cnftest-nginx"}],"workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2021-10-27T05:57:58Z"} 10.233.78.15 - - [27/Oct/2021:05:57:58 +0000] "POST /basicnf-owner/k8sregion/infra_workload HTTP/1.1" 201 304 10.233.78.15 - - [27/Oct/2021:05:57:58 +0000] "GET /basicnf-owner/k8sregion/infra_workload/hardcore_golick HTTP/1.1" 200 123 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"hardcore_golick","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2021-10-27T05:57:58Z"} 10.233.78.15 - - [27/Oct/2021:05:57:59 +0000] "POST /basicnf-owner/k8sregion/infra_workload/hardcore_golick HTTP/1.1" 405 0 10.233.78.15 - - [27/Oct/2021:05:58:27 +0000] "GET /basicnf-owner/k8sregion/infra_workload/hardcore_golick HTTP/1.1" 200 123 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"hardcore_golick","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2021-10-27T05:58:27Z"} 2021/10/27 05:58:28 [Config Cleanup] Instance hardcore_golick 2021/10/27 05:58:28 [scheduleResources]: START thread 10.233.78.15 - - [27/Oct/2021:05:58:27 +0000] "DELETE /basicnf-owner/k8sregion/infra_workload/hardcore_golick HTTP/1.1" 500 131 10.233.78.15 - - [27/Oct/2021:05:58:45 +0000] "GET /v1/rb/definition/dff8b1c9-f3e4-447f-947b-fd88d69184a4/d98dbdf5-e0d0-4982-8948-8a0aaa0f1901 HTTP/1.1" 200 270 10.233.78.15 - - [27/Oct/2021:05:58:45 +0000] "GET /v1/rb/definition/dff8b1c9-f3e4-447f-947b-fd88d69184a4/d98dbdf5-e0d0-4982-8948-8a0aaa0f1901/profile/cnftest HTTP/1.1" 200 212 2021/10/27 05:58:45 Deleted Obj with ID ObjectID("6178ea57fc11d0581157c72d") 2021/10/27 05:58:45 Deleted Obj with ID ObjectID("6178ea58fc11d0581157c72e") 2021/10/27 05:58:45 Deleted Obj with ID ObjectID("6178ea57816baef902197a20") 10.233.78.15 - - [27/Oct/2021:05:58:45 +0000] "DELETE /v1/rb/definition/dff8b1c9-f3e4-447f-947b-fd88d69184a4/d98dbdf5-e0d0-4982-8948-8a0aaa0f1901/profile/cnftest HTTP/1.1" 204 0 10.233.78.15 - - [27/Oct/2021:05:59:24 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7569 2021/10/27 05:59:24 Deleted Obj with ID ObjectID("6178e8d4fc11d0581157c71b") 10.233.78.15 - - [27/Oct/2021:05:59:24 +0000] "DELETE /v1/connectivity-info/k8sregion HTTP/1.1" 204 0 2021/10/27 05:59:24 Deleted Obj with ID ObjectID("6178e8d4816baef9021978c2") 10.233.78.15 - - [27/Oct/2021:06:01:03 +0000] "DELETE /v1/instance/amazing_sanderson HTTP/1.1" 500 177 {"error":"Getting CloudRegion Information: Get kubeconfig file: Downloading kubeconfig: Getting Connection info: Get Connection: Error finding master table: mongo: no documents in result","level":"error","msg":"Error Deleting Instance","time":"2021-10-27T06:01:03Z"}