2022/03/05 01:01:43 Starting Kubernetes Multicloud API 2022/03/05 01:01:43 Error Getting TLS Configuration. Starting without TLS... 10.233.77.220 - - [05/Mar/2022:02:16:51 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.77.220 - - [05/Mar/2022:02:18:43 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [05/Mar/2022:02:19:31 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.77.220 - - [05/Mar/2022:02:20:28 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.77.220 - - [05/Mar/2022:02:22:05 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.77.220 - - [05/Mar/2022:02:23:33 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.77.220 - - [05/Mar/2022:02:25:04 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [05/Mar/2022:02:25:32 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.77.220 - - [05/Mar/2022:02:27:18 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [05/Mar/2022:02:30:19 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.77.220 - - [05/Mar/2022:02:32:28 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.77.220 - - [05/Mar/2022:03:03:31 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.77.220 - - [05/Mar/2022:03:08:48 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.77.220 - - [05/Mar/2022:03:12:40 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.77.220 - - [05/Mar/2022:03:16:41 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.77.220 - - [05/Mar/2022:03:21:07 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.77.220 - - [05/Mar/2022:03:28:20 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 10.233.77.220 - - [05/Mar/2022:04:34:53 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.77.220 - - [05/Mar/2022:04:34:53 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7661 10.233.77.220 - - [05/Mar/2022:04:34:53 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7661 10.233.77.220 - - [05/Mar/2022:04:34:53 +0000] "POST /v1/rb/definition HTTP/1.1" 201 124 10.233.77.220 - - [05/Mar/2022:04:34:53 +0000] "GET /v1/rb/definition/cds-ms-rb-76ae77de-b70c-4798-a226-54136fae9b8e/v1 HTTP/1.1" 200 124 10.233.77.220 - - [05/Mar/2022:04:34:53 +0000] "POST /v1/rb/definition/cds-ms-rb-76ae77de-b70c-4798-a226-54136fae9b8e/v1/content HTTP/1.1" 200 0 10.233.77.220 - - [05/Mar/2022:04:34:53 +0000] "GET /v1/rb/definition/cds-ms-rb-76ae77de-b70c-4798-a226-54136fae9b8e/v1 HTTP/1.1" 200 138 10.233.77.220 - - [05/Mar/2022:04:34:53 +0000] "POST /v1/rb/definition/cds-ms-rb-76ae77de-b70c-4798-a226-54136fae9b8e/v1/profile HTTP/1.1" 201 305 10.233.77.220 - - [05/Mar/2022:04:34:53 +0000] "GET /v1/rb/definition/cds-ms-rb-76ae77de-b70c-4798-a226-54136fae9b8e/v1/profile/cds-ms-prof-c7206b91-8aca-484b-a8ee-c09a939c49cf HTTP/1.1" 200 305 10.233.77.220 - - [05/Mar/2022:04:34:53 +0000] "POST /v1/rb/definition/cds-ms-rb-76ae77de-b70c-4798-a226-54136fae9b8e/v1/profile/cds-ms-prof-c7206b91-8aca-484b-a8ee-c09a939c49cf/content HTTP/1.1" 200 0 2022/03/05 04:34:53 Error reading name cache from Database: Error finding master table: mongo: no documents in result 2022/03/05 04:34:53 Error Reading from DB: Reading cache from DB: Error finding master table: mongo: no documents in result 2022/03/05 04:34:53 Main rss info 2022/03/05 04:34:53 Path: /tmp/helm-tmpl-592076097/manifest-0 2022/03/05 04:34:53 Kind: Service 2022/03/05 04:34:53 Path: /tmp/helm-tmpl-592076097/manifest-1 2022/03/05 04:34:53 Kind: Deployment 2022/03/05 04:34:53 Crd rss info 2022/03/05 04:34:53 Hook info {"level":"info","msg":"Creating Namespace","namespace":"onap-tests","time":"2022-03-05T04:34:53Z"} 2022/03/05 04:34:53 Namespace (onap-tests) created {"filepath":"/tmp/helm-tmpl-592076097/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-05T04:34:54Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"cds-regression-mockserver","time":"2022-03-05T04:34:54Z"} {"filepath":"/tmp/helm-tmpl-592076097/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-05T04:34:54Z"} 2022/03/05 04:34:54 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"cds-cds-mockserver","time":"2022-03-05T04:34:54Z"} 10.233.77.220 - - [05/Mar/2022:04:34:53 +0000] "POST /v1/instance HTTP/1.1" 201 542 10.233.77.220 - - [05/Mar/2022:04:34:57 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7661 10.233.77.220 - - [05/Mar/2022:04:34:58 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.77.220 - - [05/Mar/2022:04:34:58 +0000] "GET /v1/rb/definition/pnf-cnf-rb-fb23fd83-156b-469e-9b0a-4c589cd95f5c/v1 HTTP/1.1" 200 125 10.233.77.220 - - [05/Mar/2022:04:34:58 +0000] "POST /v1/rb/definition/pnf-cnf-rb-fb23fd83-156b-469e-9b0a-4c589cd95f5c/v1/content HTTP/1.1" 200 0 10.233.77.220 - - [05/Mar/2022:04:34:58 +0000] "GET /v1/rb/definition/pnf-cnf-rb-fb23fd83-156b-469e-9b0a-4c589cd95f5c/v1 HTTP/1.1" 200 138 10.233.77.220 - - [05/Mar/2022:04:34:58 +0000] "POST /v1/rb/definition/pnf-cnf-rb-fb23fd83-156b-469e-9b0a-4c589cd95f5c/v1/profile HTTP/1.1" 201 314 10.233.77.220 - - [05/Mar/2022:04:34:58 +0000] "GET /v1/rb/definition/pnf-cnf-rb-fb23fd83-156b-469e-9b0a-4c589cd95f5c/v1/profile/pnf-cnf-profile-89c6b9ca-6e6b-48c1-80a2-ef49cdd604cb HTTP/1.1" 200 314 10.233.77.220 - - [05/Mar/2022:04:34:58 +0000] "POST /v1/rb/definition/pnf-cnf-rb-fb23fd83-156b-469e-9b0a-4c589cd95f5c/v1/profile/pnf-cnf-profile-89c6b9ca-6e6b-48c1-80a2-ef49cdd604cb/content HTTP/1.1" 200 0 2022/03/05 04:34:58 Main rss info 2022/03/05 04:34:58 Path: /tmp/helm-tmpl-625196585/manifest-0 2022/03/05 04:34:58 Kind: Service 2022/03/05 04:34:58 Path: /tmp/helm-tmpl-625196585/manifest-1 2022/03/05 04:34:58 Kind: Service 2022/03/05 04:34:58 Path: /tmp/helm-tmpl-625196585/manifest-2 2022/03/05 04:34:58 Kind: Pod 2022/03/05 04:34:58 Path: /tmp/helm-tmpl-625196585/manifest-3 2022/03/05 04:34:58 Kind: Deployment 2022/03/05 04:34:58 Crd rss info 2022/03/05 04:34:58 Hook info {"filepath":"/tmp/helm-tmpl-625196585/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-05T04:34:58Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-05T04:34:58Z"} {"filepath":"/tmp/helm-tmpl-625196585/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-05T04:34:58Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-05T04:34:58Z"} {"filepath":"/tmp/helm-tmpl-625196585/manifest-2","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-05T04:34:58Z"} 2022/03/05 04:34:58 No plugin for kind Pod found. Using generic Plugin 2022/03/05 04:34:58 Error converting template to map {"gvk":{"Group":"","Version":"v1","Kind":"Pod"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-03-05T04:34:58Z"} {"filepath":"/tmp/helm-tmpl-625196585/manifest-3","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-05T04:34:58Z"} 2022/03/05 04:34:58 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-03-05T04:34:58Z"} 10.233.77.220 - - [05/Mar/2022:04:34:58 +0000] "POST /v1/instance HTTP/1.1" 201 736 127.0.0.1 - - [05/Mar/2022:04:36:33 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 2022/03/05 04:37:32 [Config Cleanup] Instance suspicious_wescoff 2022/03/05 04:37:32 [scheduleResources]: START thread 2022/03/05 04:37:37 Stop config thread for suspicious_wescoff {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"warning","msg":"Deleting Resource","resource":"cds-cds-mockserver","time":"2022-03-05T04:37:37Z"} 2022/03/05 04:37:37 No plugin for kind Deployment found. Using generic Plugin 2022/03/05 04:37:37 [scheduleResources]: STOP thread {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"warning","msg":"Deleting Resource","resource":"cds-regression-mockserver","time":"2022-03-05T04:37:37Z"} 2022/03/05 04:37:37 Deleting service: cds-regression-mockserver 2022/03/05 04:37:37 Deleted Obj with ID ObjectID("6222e86d5246878e6c4d1b70") 2022/03/05 04:37:37 Deleted Obj with ID ObjectID("6222e86d268f6c07d5d9be6c") 10.233.77.220 - - [05/Mar/2022:04:37:32 +0000] "DELETE /v1/instance/suspicious_wescoff HTTP/1.1" 202 0 10.233.77.220 - - [05/Mar/2022:04:37:37 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7661 2022/03/05 04:37:37 Deleted Obj with ID ObjectID("6222e86d5246878e6c4d1b67") 10.233.77.220 - - [05/Mar/2022:04:37:37 +0000] "DELETE /v1/connectivity-info/k8sregion HTTP/1.1" 204 0 2022/03/05 04:37:37 Deleted Obj with ID ObjectID("6222e86d268f6c07d5d9be2e") 10.233.77.220 - - [05/Mar/2022:04:42:03 +0000] "DELETE /v1/instance/cranky_ramanujan HTTP/1.1" 500 177 {"error":"Getting CloudRegion Information: Get kubeconfig file: Downloading kubeconfig: Getting Connection info: Get Connection: Error finding master table: mongo: no documents in result","level":"error","msg":"Error Deleting Instance","time":"2022-03-05T04:42:03Z"} 10.233.77.220 - - [05/Mar/2022:04:42:55 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.77.220 - - [05/Mar/2022:04:42:55 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7661 10.233.77.220 - - [05/Mar/2022:04:42:55 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7661 10.233.77.220 - - [05/Mar/2022:04:42:55 +0000] "POST /v1/rb/definition HTTP/1.1" 201 124 10.233.77.220 - - [05/Mar/2022:04:42:55 +0000] "GET /v1/rb/definition/cds-ms-rb-f19c3310-0563-4b30-9012-145f7ccca1e7/v1 HTTP/1.1" 200 124 10.233.77.220 - - [05/Mar/2022:04:42:55 +0000] "POST /v1/rb/definition/cds-ms-rb-f19c3310-0563-4b30-9012-145f7ccca1e7/v1/content HTTP/1.1" 200 0 10.233.77.220 - - [05/Mar/2022:04:42:55 +0000] "GET /v1/rb/definition/cds-ms-rb-f19c3310-0563-4b30-9012-145f7ccca1e7/v1 HTTP/1.1" 200 138 10.233.77.220 - - [05/Mar/2022:04:42:55 +0000] "POST /v1/rb/definition/cds-ms-rb-f19c3310-0563-4b30-9012-145f7ccca1e7/v1/profile HTTP/1.1" 201 305 10.233.77.220 - - [05/Mar/2022:04:42:55 +0000] "GET /v1/rb/definition/cds-ms-rb-f19c3310-0563-4b30-9012-145f7ccca1e7/v1/profile/cds-ms-prof-c82f64e8-f171-4a27-be8a-b9a310efe25d HTTP/1.1" 200 305 10.233.77.220 - - [05/Mar/2022:04:42:55 +0000] "POST /v1/rb/definition/cds-ms-rb-f19c3310-0563-4b30-9012-145f7ccca1e7/v1/profile/cds-ms-prof-c82f64e8-f171-4a27-be8a-b9a310efe25d/content HTTP/1.1" 200 0 2022/03/05 04:42:55 Main rss info 2022/03/05 04:42:55 Path: /tmp/helm-tmpl-623733799/manifest-0 2022/03/05 04:42:55 Kind: Service 2022/03/05 04:42:55 Path: /tmp/helm-tmpl-623733799/manifest-1 2022/03/05 04:42:55 Kind: Deployment 2022/03/05 04:42:55 Crd rss info 2022/03/05 04:42:55 Hook info {"filepath":"/tmp/helm-tmpl-623733799/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-05T04:42:55Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"cds-regression-mockserver","time":"2022-03-05T04:42:56Z"} {"filepath":"/tmp/helm-tmpl-623733799/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-05T04:42:56Z"} 2022/03/05 04:42:56 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"cds-cds-mockserver","time":"2022-03-05T04:42:56Z"} 10.233.77.220 - - [05/Mar/2022:04:42:55 +0000] "POST /v1/instance HTTP/1.1" 201 534 10.233.77.220 - - [05/Mar/2022:04:47:10 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7661 10.233.77.220 - - [05/Mar/2022:04:47:10 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.77.220 - - [05/Mar/2022:04:47:11 +0000] "GET /v1/rb/definition/pnf-cnf-rb-ac332402-1496-450c-b07c-a34332a804a9/v1 HTTP/1.1" 200 125 10.233.77.220 - - [05/Mar/2022:04:47:11 +0000] "POST /v1/rb/definition/pnf-cnf-rb-ac332402-1496-450c-b07c-a34332a804a9/v1/content HTTP/1.1" 200 0 10.233.77.220 - - [05/Mar/2022:04:47:11 +0000] "GET /v1/rb/definition/pnf-cnf-rb-ac332402-1496-450c-b07c-a34332a804a9/v1 HTTP/1.1" 200 138 10.233.77.220 - - [05/Mar/2022:04:47:11 +0000] "POST /v1/rb/definition/pnf-cnf-rb-ac332402-1496-450c-b07c-a34332a804a9/v1/profile HTTP/1.1" 201 314 10.233.77.220 - - [05/Mar/2022:04:47:11 +0000] "GET /v1/rb/definition/pnf-cnf-rb-ac332402-1496-450c-b07c-a34332a804a9/v1/profile/pnf-cnf-profile-28111515-08f7-4ef7-8fac-58e319bf0def HTTP/1.1" 200 314 10.233.77.220 - - [05/Mar/2022:04:47:11 +0000] "POST /v1/rb/definition/pnf-cnf-rb-ac332402-1496-450c-b07c-a34332a804a9/v1/profile/pnf-cnf-profile-28111515-08f7-4ef7-8fac-58e319bf0def/content HTTP/1.1" 200 0 2022/03/05 04:47:11 Main rss info 2022/03/05 04:47:11 Path: /tmp/helm-tmpl-062802222/manifest-0 2022/03/05 04:47:11 Kind: Service 2022/03/05 04:47:11 Path: /tmp/helm-tmpl-062802222/manifest-1 2022/03/05 04:47:11 Kind: Service 2022/03/05 04:47:11 Path: /tmp/helm-tmpl-062802222/manifest-2 2022/03/05 04:47:11 Kind: Pod 2022/03/05 04:47:11 Path: /tmp/helm-tmpl-062802222/manifest-3 2022/03/05 04:47:11 Kind: Deployment 2022/03/05 04:47:11 Crd rss info 2022/03/05 04:47:11 Hook info {"filepath":"/tmp/helm-tmpl-062802222/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-03-05T04:47:12Z"} {"error":"Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","filepath":"/tmp/helm-tmpl-062802222/manifest-0","gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"error","msg":"Error Creating Resource","time":"2022-03-05T04:47:12Z"} 2022/03/05 04:47:12 Instance: condescending_mccarthy, Main rss are failed, skip post-install and remove instance in DB 2022/03/05 04:47:12 Deleted Obj with ID ObjectID("6222eb4f5246878e6c4d1b8c") 2022/03/05 04:47:12 Deleted Obj with ID ObjectID("6222eb50268f6c07d5d9c11d") 10.233.77.220 - - [05/Mar/2022:04:47:11 +0000] "POST /v1/instance HTTP/1.1" 500 180 {"error":"Create Kubernetes Resources: Error creating kind: /v1, Kind=Service: Error in plugin Service plugin: Create Service error: services \"pnf-macro-test-simulator-mongo\" already exists","level":"error","msg":"Error Creating Resource","resource":{"rb-name":"pnf-cnf-rb-ac332402-1496-450c-b07c-a34332a804a9","rb-version":"v1","profile-name":"pnf-cnf-profile-28111515-08f7-4ef7-8fac-58e319bf0def","release-name":"","cloud-region":"k8sregion","labels":{},"override-values":{}},"time":"2022-03-05T04:47:12Z"} 10.233.77.220 - - [05/Mar/2022:04:47:12 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7661 2022/03/05 04:47:12 Deleted Obj with ID ObjectID("6222ea4f5246878e6c4d1b7c") 2022/03/05 04:47:12 Deleted Obj with ID ObjectID("6222ea4f268f6c07d5d9c005") 10.233.77.220 - - [05/Mar/2022:04:47:12 +0000] "DELETE /v1/connectivity-info/k8sregion HTTP/1.1" 204 0