2022/05/25 02:55:22 Starting Kubernetes Multicloud API 2022/05/25 02:55:22 Error Getting TLS Configuration. Starting without TLS... {"level":"info","msg":"Restoring status subscription notifications","time":"2022-05-25T02:55:32Z"} {"error":"Listing Instances: Did not find any objects with tag: instance","level":"error","msg":"Error reading instance list","time":"2022-05-25T02:55:32Z"} {"error":"Listing Instances: Did not find any objects with tag: instance","level":"error","msg":"Error listing instances","profile-name":"","rb-name":"","rb-version":"","time":"2022-05-25T03:29:29Z"} 10.233.71.154 - - [25/May/2022:03:29:29 +0000] "GET /v1/instance HTTP/1.1" 500 63 10.233.71.35 - - [25/May/2022:04:05:38 +0000] "GET /v1/healthcheck HTTP/1.1" 200 0 127.0.0.1 - - [25/May/2022:04:07:04 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.71.35 - - [25/May/2022:04:20:10 +0000] "GET /v1/connectivity-info/k8sregion-pnf-macro HTTP/1.1" 500 74 10.233.71.35 - - [25/May/2022:04:20:10 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7671 10.233.71.35 - - [25/May/2022:04:20:10 +0000] "GET /v1/connectivity-info/k8sregion-pnf-macro HTTP/1.1" 200 7671 10.233.71.35 - - [25/May/2022:04:20:10 +0000] "POST /v1/rb/definition HTTP/1.1" 201 125 10.233.71.35 - - [25/May/2022:04:20:10 +0000] "GET /v1/rb/definition/pnf-cnf-rb-79e3be1e-0be6-47f4-899c-11a2a23c3b85/v1 HTTP/1.1" 200 125 10.233.71.35 - - [25/May/2022:04:20:10 +0000] "POST /v1/rb/definition/pnf-cnf-rb-79e3be1e-0be6-47f4-899c-11a2a23c3b85/v1/content HTTP/1.1" 200 0 10.233.71.35 - - [25/May/2022:04:20:10 +0000] "GET /v1/rb/definition/pnf-cnf-rb-79e3be1e-0be6-47f4-899c-11a2a23c3b85/v1 HTTP/1.1" 200 138 10.233.71.35 - - [25/May/2022:04:20:10 +0000] "POST /v1/rb/definition/pnf-cnf-rb-79e3be1e-0be6-47f4-899c-11a2a23c3b85/v1/profile HTTP/1.1" 201 314 10.233.71.35 - - [25/May/2022:04:20:10 +0000] "GET /v1/rb/definition/pnf-cnf-rb-79e3be1e-0be6-47f4-899c-11a2a23c3b85/v1/profile/pnf-cnf-profile-7e8dda71-ceea-41c1-ab99-d331297364b2 HTTP/1.1" 200 314 10.233.71.35 - - [25/May/2022:04:20:10 +0000] "POST /v1/rb/definition/pnf-cnf-rb-79e3be1e-0be6-47f4-899c-11a2a23c3b85/v1/profile/pnf-cnf-profile-7e8dda71-ceea-41c1-ab99-d331297364b2/content HTTP/1.1" 200 0 2022/05/25 04:20:10 Error reading name cache from Database: Error finding master table: mongo: no documents in result 2022/05/25 04:20:10 Error Reading from DB: Reading cache from DB: Error finding master table: mongo: no documents in result 2022/05/25 04:20:11 Main rss info 2022/05/25 04:20:11 Path: /tmp/helm-tmpl-872965293/manifest-0 2022/05/25 04:20:11 Kind: Service 2022/05/25 04:20:11 Path: /tmp/helm-tmpl-872965293/manifest-1 2022/05/25 04:20:11 Kind: Service 2022/05/25 04:20:11 Path: /tmp/helm-tmpl-872965293/manifest-2 2022/05/25 04:20:11 Kind: Pod 2022/05/25 04:20:11 Path: /tmp/helm-tmpl-872965293/manifest-3 2022/05/25 04:20:11 Kind: Deployment 2022/05/25 04:20:11 Crd rss info 2022/05/25 04:20:11 Hook info {"level":"info","msg":"Creating Namespace","namespace":"onap-tests","time":"2022-05-25T04:20:11Z"} 2022/05/25 04:20:11 Namespace (onap-tests) created {"filepath":"/tmp/helm-tmpl-872965293/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-05-25T04:20:11Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-05-25T04:20:11Z"} {"filepath":"/tmp/helm-tmpl-872965293/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-05-25T04:20:11Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-05-25T04:20:11Z"} {"filepath":"/tmp/helm-tmpl-872965293/manifest-2","level":"info","msg":"Processing Kubernetes Resource","time":"2022-05-25T04:20:11Z"} 2022/05/25 04:20:11 No plugin for kind Pod found. Using generic Plugin 2022/05/25 04:20:11 Error converting template to map {"gvk":{"Group":"","Version":"v1","Kind":"Pod"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator","time":"2022-05-25T04:20:11Z"} {"filepath":"/tmp/helm-tmpl-872965293/manifest-3","level":"info","msg":"Processing Kubernetes Resource","time":"2022-05-25T04:20:11Z"} 2022/05/25 04:20:11 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-05-25T04:20:11Z"} 10.233.71.35 - - [25/May/2022:04:20:10 +0000] "POST /v1/instance HTTP/1.1" 201 744 10.233.71.35 - - [25/May/2022:04:20:29 +0000] "GET /v1/connectivity-info/k8sregion-cds HTTP/1.1" 500 74 10.233.71.35 - - [25/May/2022:04:20:29 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7665 10.233.71.35 - - [25/May/2022:04:20:29 +0000] "GET /v1/connectivity-info/k8sregion-cds HTTP/1.1" 200 7665 10.233.71.35 - - [25/May/2022:04:20:29 +0000] "POST /v1/rb/definition HTTP/1.1" 201 124 10.233.71.35 - - [25/May/2022:04:20:30 +0000] "GET /v1/rb/definition/cds-ms-rb-f2d35998-17a3-49d0-969b-0e119d8b3c80/v1 HTTP/1.1" 200 124 10.233.71.35 - - [25/May/2022:04:20:30 +0000] "POST /v1/rb/definition/cds-ms-rb-f2d35998-17a3-49d0-969b-0e119d8b3c80/v1/content HTTP/1.1" 200 0 10.233.71.35 - - [25/May/2022:04:20:30 +0000] "GET /v1/rb/definition/cds-ms-rb-f2d35998-17a3-49d0-969b-0e119d8b3c80/v1 HTTP/1.1" 200 138 10.233.71.35 - - [25/May/2022:04:20:30 +0000] "POST /v1/rb/definition/cds-ms-rb-f2d35998-17a3-49d0-969b-0e119d8b3c80/v1/profile HTTP/1.1" 201 305 10.233.71.35 - - [25/May/2022:04:20:30 +0000] "GET /v1/rb/definition/cds-ms-rb-f2d35998-17a3-49d0-969b-0e119d8b3c80/v1/profile/cds-ms-prof-c359eb4a-2714-4790-bf39-9ac0e76c2e86 HTTP/1.1" 200 305 10.233.71.35 - - [25/May/2022:04:20:30 +0000] "POST /v1/rb/definition/cds-ms-rb-f2d35998-17a3-49d0-969b-0e119d8b3c80/v1/profile/cds-ms-prof-c359eb4a-2714-4790-bf39-9ac0e76c2e86/content HTTP/1.1" 200 0 2022/05/25 04:20:30 Main rss info 2022/05/25 04:20:30 Path: /tmp/helm-tmpl-387976029/manifest-0 2022/05/25 04:20:30 Kind: Service 2022/05/25 04:20:30 Path: /tmp/helm-tmpl-387976029/manifest-1 2022/05/25 04:20:30 Kind: Deployment 2022/05/25 04:20:30 Crd rss info 2022/05/25 04:20:30 Hook info {"filepath":"/tmp/helm-tmpl-387976029/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-05-25T04:20:30Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"cds-regression-mockserver","time":"2022-05-25T04:20:30Z"} {"filepath":"/tmp/helm-tmpl-387976029/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-05-25T04:20:30Z"} 2022/05/25 04:20:30 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"cds-cds-mockserver","time":"2022-05-25T04:20:30Z"} 10.233.71.35 - - [25/May/2022:04:20:30 +0000] "POST /v1/instance HTTP/1.1" 201 542 2022/05/25 04:21:45 [Config Cleanup] Instance clever_wozniak 2022/05/25 04:21:45 [scheduleResources]: START thread 2022/05/25 04:21:50 Stop config thread for clever_wozniak {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"warning","msg":"Deleting Resource","resource":"cds-cds-mockserver","time":"2022-05-25T04:21:50Z"} 2022/05/25 04:21:50 No plugin for kind Deployment found. Using generic Plugin 2022/05/25 04:21:50 [scheduleResources]: STOP thread {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"warning","msg":"Deleting Resource","resource":"cds-regression-mockserver","time":"2022-05-25T04:21:50Z"} 2022/05/25 04:21:50 Deleting service: cds-regression-mockserver {"instance":"clever_wozniak","level":"info","msg":"[START] status notify thread for ","time":"2022-05-25T04:21:50Z"} {"instance":"clever_wozniak","level":"info","msg":"[STOP] status notify thread for ","time":"2022-05-25T04:21:55Z"} 2022/05/25 04:21:55 Deleted Obj with ID ObjectID("628dae8e4795edc8bf14e820") 2022/05/25 04:21:55 Deleted Obj with ID ObjectID("628dae8e06606ef2bde4d210") 10.233.71.35 - - [25/May/2022:04:21:45 +0000] "DELETE /v1/instance/clever_wozniak HTTP/1.1" 202 0 10.233.71.35 - - [25/May/2022:04:21:55 +0000] "GET /v1/connectivity-info/k8sregion-cds HTTP/1.1" 200 7665 2022/05/25 04:21:55 Deleted Obj with ID ObjectID("628dae8d4795edc8bf14e818") 2022/05/25 04:21:55 Deleted Obj with ID ObjectID("628dae8e06606ef2bde4d1d5") 10.233.71.35 - - [25/May/2022:04:21:55 +0000] "DELETE /v1/connectivity-info/k8sregion-cds HTTP/1.1" 204 0 10.233.71.35 - - [25/May/2022:04:22:12 +0000] "GET /v1/connectivity-info/k8sregion-cnf-macro HTTP/1.1" 500 74 10.233.71.35 - - [25/May/2022:04:22:12 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7678 10.233.71.35 - - [25/May/2022:04:22:12 +0000] "GET /v1/connectivity-info/k8sregion-cnf-macro HTTP/1.1" 200 7678 127.0.0.1 - - [25/May/2022:04:22:27 +0000] "POST /v1/rb/definition HTTP/1.1" 201 286 127.0.0.1 - - [25/May/2022:04:22:27 +0000] "POST /v1/rb/definition HTTP/1.1" 201 271 127.0.0.1 - - [25/May/2022:04:22:27 +0000] "POST /v1/rb/definition/38ede1ca-10c3-479a-9027-9b24fbd1108f/6aaff6cd-0ef4-4572-bd59-5ee1ebc73d7e/content HTTP/1.1" 200 0 127.0.0.1 - - [25/May/2022:04:22:45 +0000] "POST /v1/rb/definition HTTP/1.1" 201 270 10.233.71.35 - - [25/May/2022:04:22:48 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 500 74 10.233.71.35 - - [25/May/2022:04:22:48 +0000] "POST /v1/connectivity-info HTTP/1.1" 201 7661 10.233.71.35 - - [25/May/2022:04:22:48 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7661 127.0.0.1 - - [25/May/2022:04:23:07 +0000] "POST /v1/rb/definition HTTP/1.1" 201 265 127.0.0.1 - - [25/May/2022:04:23:07 +0000] "POST /v1/rb/definition/159ed351-89f0-42a3-9218-d6c0dfa1429d/7d14e583-0809-49f4-aa09-ec4064dc6ef7/content HTTP/1.1" 200 0 127.0.0.1 - - [25/May/2022:04:23:33 +0000] "POST /v1/rb/definition HTTP/1.1" 201 267 10.233.66.156 - - [25/May/2022:04:28:36 +0000] "GET /v1/rb/definition/38ede1ca-10c3-479a-9027-9b24fbd1108f/6aaff6cd-0ef4-4572-bd59-5ee1ebc73d7e HTTP/1.1" 200 277 10.233.66.156 - - [25/May/2022:04:28:36 +0000] "GET /v1/rb/definition/38ede1ca-10c3-479a-9027-9b24fbd1108f/6aaff6cd-0ef4-4572-bd59-5ee1ebc73d7e/profile/node-port-profile HTTP/1.1" 404 87 10.233.66.156 - - [25/May/2022:04:28:36 +0000] "POST /v1/rb/definition/38ede1ca-10c3-479a-9027-9b24fbd1108f/6aaff6cd-0ef4-4572-bd59-5ee1ebc73d7e/profile HTTP/1.1" 201 270 10.233.66.156 - - [25/May/2022:04:28:36 +0000] "POST /v1/rb/definition/38ede1ca-10c3-479a-9027-9b24fbd1108f/6aaff6cd-0ef4-4572-bd59-5ee1ebc73d7e/profile/node-port-profile/content HTTP/1.1" 200 0 10.233.66.156 - - [25/May/2022:04:28:47 +0000] "GET /v1/rb/definition/38ede1ca-10c3-479a-9027-9b24fbd1108f/6aaff6cd-0ef4-4572-bd59-5ee1ebc73d7e HTTP/1.1" 200 277 10.233.66.156 - - [25/May/2022:04:28:47 +0000] "GET /v1/rb/definition/38ede1ca-10c3-479a-9027-9b24fbd1108f/6aaff6cd-0ef4-4572-bd59-5ee1ebc73d7e/config-template/replica-count-template HTTP/1.1" 500 78 2022/05/25 04:28:47 [ConfigiTemplate]: create 38ede1ca-10c3-479a-9027-9b24fbd1108f 10.233.66.156 - - [25/May/2022:04:28:47 +0000] "POST /v1/rb/definition/38ede1ca-10c3-479a-9027-9b24fbd1108f/6aaff6cd-0ef4-4572-bd59-5ee1ebc73d7e/config-template HTTP/1.1" 201 113 2022/05/25 04:28:47 [ConfigTemplate]: Upload replica-count-template 10.233.66.156 - - [25/May/2022:04:28:47 +0000] "POST /v1/rb/definition/38ede1ca-10c3-479a-9027-9b24fbd1108f/6aaff6cd-0ef4-4572-bd59-5ee1ebc73d7e/config-template/replica-count-template/content HTTP/1.1" 200 0 2022/05/25 04:28:50 Main rss info 2022/05/25 04:28:50 Path: /tmp/helm-tmpl-125717669/manifest-0 2022/05/25 04:28:50 Kind: Service 2022/05/25 04:28:50 Path: /tmp/helm-tmpl-125717669/manifest-1 2022/05/25 04:28:50 Kind: Deployment 2022/05/25 04:28:50 Crd rss info 2022/05/25 04:28:50 Hook info {"filepath":"/tmp/helm-tmpl-125717669/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-05-25T04:28:50Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"rel-1-apache","time":"2022-05-25T04:28:50Z"} {"filepath":"/tmp/helm-tmpl-125717669/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-05-25T04:28:50Z"} 2022/05/25 04:28:50 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"rel-1-apache","time":"2022-05-25T04:28:50Z"} 10.233.71.154 - - [25/May/2022:04:28:50 +0000] "POST /v1/instance HTTP/1.1" 201 1287 2022/05/25 04:28:51 Deployment is not ready: onap-tests/rel-1-apache. 0 out of 1 expected pods are ready 10.233.71.154 - - [25/May/2022:04:28:51 +0000] "GET /v1/instance/affectionate_khorana/status HTTP/1.1" 200 14563 {"instance":"affectionate_khorana","level":"info","msg":"[START] status notify thread for ","time":"2022-05-25T04:29:16Z"} {"instance":"affectionate_khorana","level":"info","msg":"REFRESH WATCHERS","subscription":"affectionate_khorana-cnf-adapter","time":"2022-05-25T04:29:21Z"} {"Instance":"affectionate_khorana","Kind":"Service","Subscription":"affectionate_khorana-cnf-adapter","level":"info","msg":"[START] Watcher","time":"2022-05-25T04:29:21Z"} {"Instance":"affectionate_khorana","Kind":"Pod","Subscription":"affectionate_khorana-cnf-adapter","level":"info","msg":"[START] Watcher","time":"2022-05-25T04:29:21Z"} {"Instance":"affectionate_khorana","Kind":"Deployment","Subscription":"affectionate_khorana-cnf-adapter","level":"info","msg":"[START] Watcher","time":"2022-05-25T04:29:21Z"} {"InstanceId":"affectionate_khorana","SubscriptionName":"affectionate_khorana-cnf-adapter","level":"info","msg":"Successfully created Status Subscription","time":"2022-05-25T04:29:21Z"} 10.233.71.154 - - [25/May/2022:04:29:16 +0000] "POST /v1/instance/affectionate_khorana/status/subscription HTTP/1.1" 201 666 2022/05/25 04:29:25 Deployment is not ready: onap-tests/rel-1-apache. 0 out of 1 expected pods are ready 10.233.66.156 - - [25/May/2022:04:29:25 +0000] "GET /v1/instance/affectionate_khorana/status HTTP/1.1" 200 14903 {"InstanceId":"affectionate_khorana","SubscriptionName":"affectionate_khorana-cnf-adapter","level":"info","msg":"[START] status verification thread","time":"2022-05-25T04:29:26Z"} 10.233.66.156 - - [25/May/2022:04:29:35 +0000] "GET /v1/instance/affectionate_khorana/status HTTP/1.1" 200 14715 2022/05/25 04:29:35 [scheduleResources]: START thread 10.233.66.156 - - [25/May/2022:04:29:35 +0000] "GET /v1/instance/affectionate_khorana/config/replica-count-change HTTP/1.1" 500 60 2022/05/25 04:29:41 [Config Create] Instance affectionate_khorana 2022/05/25 04:29:41 [ConfigTemplate]: Download replica-count-template 2022/05/25 04:29:41 Before Sent to goroutine {38ede1ca-10c3-479a-9027-9b24fbd1108f 6aaff6cd-0ef4-4572-bd59-5ee1ebc73d7e node-port-profile node-port-profile onap-tests 1.19.0 map[] []} 2022/05/25 04:29:41 Message Sent to goroutine {38ede1ca-10c3-479a-9027-9b24fbd1108f 6aaff6cd-0ef4-4572-bd59-5ee1ebc73d7e node-port-profile node-port-profile onap-tests 1.19.0 map[] []} 2022/05/25 04:29:41 [scheduleResources]: POST {38ede1ca-10c3-479a-9027-9b24fbd1108f 6aaff6cd-0ef4-4572-bd59-5ee1ebc73d7e node-port-profile node-port-profile onap-tests 1.19.0 map[] []} [{apps/v1, Kind=Deployment /tmp/helm-tmpl-621845415/manifest-0}] {"filepath":"/tmp/helm-tmpl-621845415/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-05-25T04:29:41Z"} 2022/05/25 04:29:41 No plugin for kind Deployment found. Using generic Plugin {"error":"Create object error: deployments.apps \"rel-1-apache\" already exists","filepath":"/tmp/helm-tmpl-621845415/manifest-0","gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"error","msg":"Error Creating Resource","time":"2022-05-25T04:29:41Z"} {"filepath":"/tmp/helm-tmpl-621845415/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-05-25T04:29:41Z"} 2022/05/25 04:29:41 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Updated Kubernetes Resource","resource":"rel-1-apache","time":"2022-05-25T04:29:41Z"} 2022/05/25 04:29:41 POST result: [{{apps/v1, Kind=Deployment rel-1-apache} UPDATED}] 10.233.66.156 - - [25/May/2022:04:29:41 +0000] "POST /v1/instance/affectionate_khorana/config HTTP/1.1" 201 272 2022/05/25 04:29:41 Deployment is not ready: onap-tests/rel-1-apache. 1 out of 2 expected pods are ready 10.233.66.156 - - [25/May/2022:04:29:41 +0000] "GET /v1/instance/affectionate_khorana/status HTTP/1.1" 200 20962 2022/05/25 04:29:51 Deployment is not ready: onap-tests/rel-1-apache. 1 out of 2 expected pods are ready 10.233.66.156 - - [25/May/2022:04:29:51 +0000] "GET /v1/instance/affectionate_khorana/status HTTP/1.1" 200 20962 {"Instance":"affectionate_khorana","Subscription":"affectionate_khorana-cnf-adapter","level":"info","msg":"CHANGE DETECTED","time":"2022-05-25T04:30:01Z"} {"Kind":"Pod","Name":"rel-1-apache-66b679cfc-xxqw9","level":"info","msg":"CREATED","time":"2022-05-25T04:30:01Z"} {"Kind":"Deployment","Name":"rel-1-apache","level":"info","msg":"MODIFIED","time":"2022-05-25T04:30:01Z"} 2022/05/25 04:30:01 Deployment is not ready: onap-tests/rel-1-apache. 1 out of 2 expected pods are ready 10.233.71.154 - - [25/May/2022:04:30:01 +0000] "GET /v1/instance/affectionate_khorana/status HTTP/1.1" 200 21305 2022/05/25 04:30:01 Deployment is not ready: onap-tests/rel-1-apache. 1 out of 2 expected pods are ready 10.233.66.156 - - [25/May/2022:04:30:01 +0000] "GET /v1/instance/affectionate_khorana/status HTTP/1.1" 200 21305 {"InstanceId":"affectionate_khorana","Result":202,"SubscriptionName":"affectionate_khorana-cnf-adapter","level":"info","msg":"Notification sent","time":"2022-05-25T04:30:04Z"} 2022/05/25 04:30:11 Deployment is not ready: onap-tests/rel-1-apache. 1 out of 2 expected pods are ready 10.233.66.156 - - [25/May/2022:04:30:11 +0000] "GET /v1/instance/affectionate_khorana/status HTTP/1.1" 200 21305 2022/05/25 04:30:21 Deployment is not ready: onap-tests/rel-1-apache. 1 out of 2 expected pods are ready 10.233.66.156 - - [25/May/2022:04:30:21 +0000] "GET /v1/instance/affectionate_khorana/status HTTP/1.1" 200 21305 10.233.66.156 - - [25/May/2022:04:30:31 +0000] "GET /v1/instance/affectionate_khorana/status HTTP/1.1" 200 21013 {"Instance":"affectionate_khorana","Subscription":"affectionate_khorana-cnf-adapter","level":"info","msg":"CHANGE DETECTED","time":"2022-05-25T04:30:34Z"} {"Kind":"Deployment","Name":"rel-1-apache","level":"info","msg":"MODIFIED","time":"2022-05-25T04:30:34Z"} {"Kind":"Pod","Name":"rel-1-apache-66b679cfc-xxqw9","level":"info","msg":"MODIFIED","time":"2022-05-25T04:30:34Z"} 10.233.71.154 - - [25/May/2022:04:30:34 +0000] "GET /v1/instance/affectionate_khorana/status HTTP/1.1" 200 21013 {"InstanceId":"affectionate_khorana","Result":202,"SubscriptionName":"affectionate_khorana-cnf-adapter","level":"info","msg":"Notification sent","time":"2022-05-25T04:30:36Z"} {"Instance":"affectionate_khorana","Kind":"Service","Subscription":"affectionate_khorana-cnf-adapter","level":"info","msg":"[STOP] Watcher","time":"2022-05-25T04:30:53Z"} {"Instance":"affectionate_khorana","Kind":"Pod","Subscription":"affectionate_khorana-cnf-adapter","level":"info","msg":"[STOP] Watcher","time":"2022-05-25T04:30:53Z"} {"Instance":"affectionate_khorana","Kind":"Deployment","Subscription":"affectionate_khorana-cnf-adapter","level":"info","msg":"[STOP] Watcher","time":"2022-05-25T04:30:53Z"} 2022/05/25 04:30:53 Deleted Obj with ID ObjectID("628db0a14795edc8bf14e83c") 2022/05/25 04:30:53 Deleted Obj with ID ObjectID("628db0a106606ef2bde4d415") 10.233.71.154 - - [25/May/2022:04:30:53 +0000] "DELETE /v1/instance/affectionate_khorana/status/subscription/affectionate_khorana-cnf-adapter HTTP/1.1" 202 0 2022/05/25 04:30:53 [Config Cleanup] Instance affectionate_khorana 2022/05/25 04:30:53 [Config Delete] Instance affectionate_khorana Config replica-count-change 2022/05/25 04:30:53 [ConfigTemplate]: Download replica-count-template 2022/05/25 04:30:53 Before Sent to goroutine {38ede1ca-10c3-479a-9027-9b24fbd1108f 6aaff6cd-0ef4-4572-bd59-5ee1ebc73d7e node-port-profile node-port-profile onap-tests 1.19.0 map[] []} 2022/05/25 04:30:53 Message Sent to goroutine {38ede1ca-10c3-479a-9027-9b24fbd1108f 6aaff6cd-0ef4-4572-bd59-5ee1ebc73d7e node-port-profile node-port-profile onap-tests 1.19.0 map[] []} 2022/05/25 04:30:53 [scheduleResources]: DELETE {38ede1ca-10c3-479a-9027-9b24fbd1108f 6aaff6cd-0ef4-4572-bd59-5ee1ebc73d7e node-port-profile node-port-profile onap-tests 1.19.0 map[] []} [{{apps/v1, Kind=Deployment rel-1-apache} UPDATED}] {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"warning","msg":"Deleting Resource","resource":"rel-1-apache","time":"2022-05-25T04:30:53Z"} 2022/05/25 04:30:53 No plugin for kind Deployment found. Using generic Plugin 2022/05/25 04:30:53 DELETE resources: [[{{apps/v1, Kind=Deployment rel-1-apache} UPDATED}]] 2022/05/25 04:30:53 [Config Delete All] Instance affectionate_khorana Config replica-count-change 2022/05/25 04:30:53 Stop config thread for affectionate_khorana {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"warning","msg":"Deleting Resource","resource":"rel-1-apache","time":"2022-05-25T04:30:53Z"} 2022/05/25 04:30:53 No plugin for kind Deployment found. Using generic Plugin 2022/05/25 04:30:53 [scheduleResources]: STOP thread {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"warning","msg":"Resource already does not exist","resource":"rel-1-apache","time":"2022-05-25T04:30:53Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"warning","msg":"Deleting Resource","resource":"rel-1-apache","time":"2022-05-25T04:30:53Z"} 2022/05/25 04:30:53 Deleting service: rel-1-apache {"instance":"affectionate_khorana","level":"info","msg":"[STOP] status notify thread for ","time":"2022-05-25T04:30:53Z"} 2022/05/25 04:30:53 Deleted Obj with ID ObjectID("628db0824795edc8bf14e83b") 2022/05/25 04:30:53 Deleted Obj with ID ObjectID("628db08206606ef2bde4d3ed") 10.233.71.154 - - [25/May/2022:04:30:53 +0000] "DELETE /v1/instance/affectionate_khorana HTTP/1.1" 202 0 {"instance":"affectionate_khorana","level":"info","msg":"[START] status notify thread for ","time":"2022-05-25T04:31:06Z"} 10.233.71.35 - - [25/May/2022:04:31:11 +0000] "GET /v1/connectivity-info/k8sregion-cnf-macro HTTP/1.1" 200 7678 2022/05/25 04:31:11 Deleted Obj with ID ObjectID("628daef44795edc8bf14e821") 2022/05/25 04:31:11 Deleted Obj with ID ObjectID("628daef406606ef2bde4d270") 10.233.71.35 - - [25/May/2022:04:31:11 +0000] "DELETE /v1/connectivity-info/k8sregion-cnf-macro HTTP/1.1" 204 0 {"InstanceId":"affectionate_khorana","SubscriptionName":"affectionate_khorana-cnf-adapter","level":"info","msg":"[STOP] status verification thread","time":"2022-05-25T04:31:11Z"} 10.233.71.35 - - [25/May/2022:04:31:30 +0000] "GET /v1/rb/definition/159ed351-89f0-42a3-9218-d6c0dfa1429d/7d14e583-0809-49f4-aa09-ec4064dc6ef7 HTTP/1.1" 200 270 10.233.71.35 - - [25/May/2022:04:31:30 +0000] "GET /v1/rb/definition/159ed351-89f0-42a3-9218-d6c0dfa1429d/7d14e583-0809-49f4-aa09-ec4064dc6ef7/profile/cnftest HTTP/1.1" 404 87 10.233.71.35 - - [25/May/2022:04:31:30 +0000] "POST /v1/rb/definition/159ed351-89f0-42a3-9218-d6c0dfa1429d/7d14e583-0809-49f4-aa09-ec4064dc6ef7/profile HTTP/1.1" 201 240 10.233.71.35 - - [25/May/2022:04:31:30 +0000] "GET /v1/rb/definition/159ed351-89f0-42a3-9218-d6c0dfa1429d/7d14e583-0809-49f4-aa09-ec4064dc6ef7/profile/cnftest HTTP/1.1" 200 240 10.233.71.35 - - [25/May/2022:04:31:30 +0000] "POST /v1/rb/definition/159ed351-89f0-42a3-9218-d6c0dfa1429d/7d14e583-0809-49f4-aa09-ec4064dc6ef7/profile/cnftest/content HTTP/1.1" 200 0 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"","workload_status":"GET_COMPLETE","workload_status_reason":{"stacks":[]}},"time":"2022-05-25T04:31:40Z"} 10.233.71.35 - - [25/May/2022:04:31:40 +0000] "GET /basicnf-owner/k8sregion/infra_workload?name=basic_cnf-55ed7c6e-7a87-4ebd-a5ec-e210ed065cd2 HTTP/1.1" 200 114 {"Body":{},"Payload":{"generic-vnf-id":"6d36ffdc-087c-45fd-9c31-274bf9721f7d","vf-module-id":"088952ef-7045-4e37-b4d4-a8d863ea56b3","vf-module-model-invariant-id":"159ed351-89f0-42a3-9218-d6c0dfa1429d","vf-module-model-version-id":"fa78db6a-c16e-49d9-be45-7d93aa597c7c","vf-module-model-customization-id":"7d14e583-0809-49f4-aa09-ec4064dc6ef7","oof_directives":{"attributes":null},"sdnc_directives":{"attributes":[{"attribute_name":"k8s-rb-profile-name","attribute_value":"cnftest"},{"attribute_name":"k8s-rb-profile-namespace","attribute_value":"k8s"},{"attribute_name":"availability_zone_0","attribute_value":"basicnf-availability-zone"},{"attribute_name":"dummy_name_0","attribute_value":"dummy_name"}]},"user_directives":{"attributes":null},"template_type":"","template_data":{"stack_name":"basic_cnf-55ed7c6e-7a87-4ebd-a5ec-e210ed065cd2"}},"level":"info","msg":"Broker API Payload","time":"2022-05-25T04:31:40Z"} {"level":"info","msg":"Instance API Payload","payload":{"rb-name":"159ed351-89f0-42a3-9218-d6c0dfa1429d","rb-version":"7d14e583-0809-49f4-aa09-ec4064dc6ef7","profile-name":"cnftest","release-name":"088952ef-7045-4e37-b4d4-a8d863ea56b3","cloud-region":"k8sregion","labels":{"stack-name":"basic_cnf-55ed7c6e-7a87-4ebd-a5ec-e210ed065cd2"},"override-values":{"availability_zone_0":"basicnf-availability-zone","dummy_name_0":"dummy_name","k8s-rb-profile-name":"cnftest","k8s-rb-profile-namespace":"k8s"}},"time":"2022-05-25T04:31:40Z"} 2022/05/25 04:31:40 Main rss info 2022/05/25 04:31:40 Path: /tmp/helm-tmpl-472567464/manifest-0 2022/05/25 04:31:40 Kind: Service 2022/05/25 04:31:40 Path: /tmp/helm-tmpl-472567464/manifest-1 2022/05/25 04:31:40 Kind: Deployment 2022/05/25 04:31:40 Crd rss info 2022/05/25 04:31:40 Hook info {"level":"info","msg":"Creating Namespace","namespace":"k8s","time":"2022-05-25T04:31:40Z"} 2022/05/25 04:31:40 Namespace (k8s) created {"filepath":"/tmp/helm-tmpl-472567464/manifest-0","level":"info","msg":"Processing Kubernetes Resource","time":"2022-05-25T04:31:40Z"} {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-05-25T04:31:40Z"} {"filepath":"/tmp/helm-tmpl-472567464/manifest-1","level":"info","msg":"Processing Kubernetes Resource","time":"2022-05-25T04:31:40Z"} 2022/05/25 04:31:40 No plugin for kind Deployment found. Using generic Plugin {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"info","msg":"Created Kubernetes Resource","resource":"cnftest-nginx","time":"2022-05-25T04:31:40Z"} {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"gallant_wilson","template_response":[{"GVK":{"Group":"","Version":"v1","Kind":"Service"},"Name":"cnftest-nginx"},{"GVK":{"Group":"apps","Version":"v1","Kind":"Deployment"},"Name":"cnftest-nginx"}],"workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-05-25T04:31:40Z"} 10.233.71.35 - - [25/May/2022:04:31:40 +0000] "POST /basicnf-owner/k8sregion/infra_workload HTTP/1.1" 201 303 10.233.71.35 - - [25/May/2022:04:31:41 +0000] "GET /basicnf-owner/k8sregion/infra_workload/gallant_wilson HTTP/1.1" 200 122 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"gallant_wilson","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-05-25T04:31:41Z"} 10.233.71.35 - - [25/May/2022:04:31:41 +0000] "POST /basicnf-owner/k8sregion/infra_workload/gallant_wilson HTTP/1.1" 405 0 2022/05/25 04:32:00 [Config Cleanup] Instance loving_burnell 2022/05/25 04:32:00 [scheduleResources]: START thread 2022/05/25 04:32:05 Stop config thread for loving_burnell {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"warning","msg":"Deleting Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-05-25T04:32:05Z"} 2022/05/25 04:32:05 No plugin for kind Deployment found. Using generic Plugin 2022/05/25 04:32:05 [scheduleResources]: STOP thread {"gvk":{"Group":"","Version":"v1","Kind":"Pod"},"level":"warning","msg":"Deleting Resource","resource":"pnf-macro-test-simulator","time":"2022-05-25T04:32:05Z"} 2022/05/25 04:32:05 No plugin for kind Pod found. Using generic Plugin {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"warning","msg":"Deleting Resource","resource":"pnf-macro-test-simulator","time":"2022-05-25T04:32:05Z"} 2022/05/25 04:32:05 Deleting service: pnf-macro-test-simulator {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"warning","msg":"Deleting Resource","resource":"pnf-macro-test-simulator-mongo","time":"2022-05-25T04:32:05Z"} 2022/05/25 04:32:05 Deleting service: pnf-macro-test-simulator-mongo {"instance":"loving_burnell","level":"info","msg":"[START] status notify thread for ","time":"2022-05-25T04:32:05Z"} {"instance":"loving_burnell","level":"info","msg":"[STOP] status notify thread for ","time":"2022-05-25T04:32:10Z"} 2022/05/25 04:32:10 Deleted Obj with ID ObjectID("628dae7b4795edc8bf14e817") 2022/05/25 04:32:10 Deleted Obj with ID ObjectID("628dae7b06606ef2bde4d1b9") 10.233.71.35 - - [25/May/2022:04:31:59 +0000] "DELETE /v1/instance/loving_burnell HTTP/1.1" 202 0 10.233.71.35 - - [25/May/2022:04:32:10 +0000] "GET /v1/connectivity-info/k8sregion-pnf-macro HTTP/1.1" 200 7671 2022/05/25 04:32:10 Deleted Obj with ID ObjectID("628dae7a4795edc8bf14e80e") 2022/05/25 04:32:10 Deleted Obj with ID ObjectID("628dae7a06606ef2bde4d17b") 10.233.71.35 - - [25/May/2022:04:32:10 +0000] "DELETE /v1/connectivity-info/k8sregion-pnf-macro HTTP/1.1" 204 0 10.233.71.35 - - [25/May/2022:04:32:14 +0000] "GET /basicnf-owner/k8sregion/infra_workload/gallant_wilson HTTP/1.1" 200 122 {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"gallant_wilson","workload_status":"CREATE_COMPLETE","workload_status_reason":null},"time":"2022-05-25T04:32:14Z"} 2022/05/25 04:32:14 [Config Cleanup] Instance gallant_wilson 2022/05/25 04:32:14 [scheduleResources]: START thread 2022/05/25 04:32:19 Stop config thread for gallant_wilson {"gvk":{"Group":"apps","Version":"v1","Kind":"Deployment"},"level":"warning","msg":"Deleting Resource","resource":"cnftest-nginx","time":"2022-05-25T04:32:19Z"} 2022/05/25 04:32:19 No plugin for kind Deployment found. Using generic Plugin 2022/05/25 04:32:19 [scheduleResources]: STOP thread {"gvk":{"Group":"","Version":"v1","Kind":"Service"},"level":"warning","msg":"Deleting Resource","resource":"cnftest-nginx","time":"2022-05-25T04:32:19Z"} 2022/05/25 04:32:19 Deleting service: cnftest-nginx {"instance":"gallant_wilson","level":"info","msg":"[START] status notify thread for ","time":"2022-05-25T04:32:19Z"} {"instance":"gallant_wilson","level":"info","msg":"[STOP] status notify thread for ","time":"2022-05-25T04:32:24Z"} 2022/05/25 04:32:24 Deleted Obj with ID ObjectID("628db12c4795edc8bf14e83f") 2022/05/25 04:32:24 Deleted Obj with ID ObjectID("628db12c06606ef2bde4d554") {"level":"info","msg":"Broker API Response","response":{"template_type":"heat","workload_id":"gallant_wilson","workload_status":"DELETE_COMPLETE","workload_status_reason":null},"time":"2022-05-25T04:32:24Z"} 10.233.71.35 - - [25/May/2022:04:32:14 +0000] "DELETE /basicnf-owner/k8sregion/infra_workload/gallant_wilson HTTP/1.1" 202 122 10.233.71.35 - - [25/May/2022:04:32:40 +0000] "GET /v1/rb/definition/159ed351-89f0-42a3-9218-d6c0dfa1429d/7d14e583-0809-49f4-aa09-ec4064dc6ef7 HTTP/1.1" 200 270 10.233.71.35 - - [25/May/2022:04:32:40 +0000] "GET /v1/rb/definition/159ed351-89f0-42a3-9218-d6c0dfa1429d/7d14e583-0809-49f4-aa09-ec4064dc6ef7/profile/cnftest HTTP/1.1" 200 240 2022/05/25 04:32:40 Deleted Obj with ID ObjectID("628db1224795edc8bf14e83d") 2022/05/25 04:32:40 Deleted Obj with ID ObjectID("628db1224795edc8bf14e83e") 10.233.71.35 - - [25/May/2022:04:32:40 +0000] "DELETE /v1/rb/definition/159ed351-89f0-42a3-9218-d6c0dfa1429d/7d14e583-0809-49f4-aa09-ec4064dc6ef7/profile/cnftest HTTP/1.1" 204 0 2022/05/25 04:32:40 Deleted Obj with ID ObjectID("628db12206606ef2bde4d531") 10.233.71.35 - - [25/May/2022:04:33:08 +0000] "GET /v1/connectivity-info/k8sregion HTTP/1.1" 200 7661 2022/05/25 04:33:08 Deleted Obj with ID ObjectID("628daf184795edc8bf14e82d") 10.233.71.35 - - [25/May/2022:04:33:08 +0000] "DELETE /v1/connectivity-info/k8sregion HTTP/1.1" 204 0 2022/05/25 04:33:08 Deleted Obj with ID ObjectID("628daf1806606ef2bde4d2b9")