SuccessConsole Output

Skipping 672 KB.. Full Log
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 290 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 5b92d558-c240-4fa7-84e7-dab4afb419f2
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:5b92d558-c240-4fa7-84e7-dab4afb419f2' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:5b92d558-c240-4fa7-84e7-dab4afb419f2'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:5b92d558-c240-4fa7-84e7-dab4afb419f2'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 289 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 5b92d558-c240-4fa7-84e7-dab4afb419f2
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:5b92d558-c240-4fa7-84e7-dab4afb419f2' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:5b92d558-c240-4fa7-84e7-dab4afb419f2'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:5b92d558-c240-4fa7-84e7-dab4afb419f2'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 289 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 5b92d558-c240-4fa7-84e7-dab4afb419f2
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 5b92d558-c240-4fa7-84e7-dab4afb419f2
++ espod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
++ myproject=.operations
++ mymessage=57f538b0-5eb3-41fb-8545-f99be8df52a2
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk .operations _count systemd.u.SYSLOG_IDENTIFIER 57f538b0-5eb3-41fb-8545-f99be8df52a2
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:57f538b0-5eb3-41fb-8545-f99be8df52a2' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:57f538b0-5eb3-41fb-8545-f99be8df52a2'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-n5j3wlaj-1-h1fvk -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:57f538b0-5eb3-41fb-8545-f99be8df52a2'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 57f538b0-5eb3-41fb-8545-f99be8df52a2
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 57f538b0-5eb3-41fb-8545-f99be8df52a2
++ '[' -n '' ']'
++ '[' -n '' ']'
+++ date +%s
++ local endtime=1497028310
+++ expr 1497028310 - 1497028292
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 18 seconds at 2017-06-09 17:11:50.456643357+00:00
++ echo END wait_for_fluentd_to_catch_up took 18 seconds at 2017-06-09 17:11:50.456643357+00:00
++ return 0
++ return 0
++ cleanup
++ cleanup_forward
++ oc label node --all logging-infra-fluentd-
node "172.18.7.3" labeled
++ wait_for_pod_ACTION stop logging-fluentd-31277
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-31277
++ '[' -z logging-fluentd-31277 -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-31277
pod logging-fluentd-31277 still running
++ '[' -n 1 ']'
++ echo pod logging-fluentd-31277 still running
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' stop = start ']'
++ '[' 110 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-31277
++ '[' stop = start ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
++ oc delete daemonset/logging-forward-fluentd
Error from server (NotFound): daemonsets.extensions "logging-forward-fluentd" not found
++ :
+++ oc get configmap/logging-fluentd -o yaml
+++ grep '<match \*\*>'
++ '[' -n '' ']'
++ oc patch configmap/logging-fluentd --type=json --patch '[{ "op": "replace", "path": "/data/secure-forward.conf", "value": "\
# @type secure_forward\n\
# self_hostname forwarding-${HOSTNAME}\n\
# shared_key aggregated_logging_ci_testing\n\
#  secure no\n\
#  <server>\n\
#   host ${FLUENTD_FORWARD}\n\
#   port 24284\n\
#  </server>"}]'
configmap "logging-fluentd" not patched
++ oc label node --all logging-infra-fluentd=true
node "172.18.7.3" labeled
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
pod for component=fluentd not running yet
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-k81b9
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-k81b9 ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-k81b9
++ oc get events -o yaml
running test test-json-parsing.sh
++ set -o nounset
++ set -o pipefail
++ type get_running_pod
++ ARTIFACT_DIR=/tmp/origin-aggregated-logging/artifacts
++ '[' '!' -d /tmp/origin-aggregated-logging/artifacts ']'
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ es_pod=logging-es-data-master-oawpjydu-1-f676p
+++ uuidgen
++ uuid_es=86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
Adding test message 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb to Kibana . . .
++ echo Adding test message 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb to Kibana . . .
++ add_test_message 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-nxzg9
++ oc exec logging-kibana-1-nxzg9 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
Waiting 600 seconds for 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb to show up in Elasticsearch . . .
++ rc=0
++ timeout=600
++ echo Waiting 600 seconds for 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb to show up in Elasticsearch . . .
++ espod=logging-es-data-master-oawpjydu-1-f676p
++ myproject=project.logging.
++ mymessage=86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 600
++ let ii=600
++ local interval=1
++ '[' 600 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging. _count message 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 599 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging. _count message 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 598 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging. _count message 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 597 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging. _count message 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 596 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging. _count message 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 595 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging. _count message 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 594 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging. _count message 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 593 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging. _count message 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 592 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging. _count message 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 591 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging. _count message 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 590 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging. _count message 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 589 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging. _count message 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
good - ./logging.sh: found 1 record project logging for 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 589 -le 0 ']'
++ return 0
++ echo good - ./logging.sh: found 1 record project logging for 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
Testing if record is in correct format . . .
++ echo Testing if record is in correct format . . .
++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging. _search message 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
++ python test-json-parsing.py 86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb
++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging.*/_search?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb' --connect-timeout 1
++ local pod=logging-es-data-master-oawpjydu-1-f676p
++ local 'endpoint=/project.logging.*/_search?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
++ shift
++ shift
++ args=("${@:-}")
++ local args
++ local secret_dir=/etc/elasticsearch/secret/
++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_search?q=message:86c5a0ac-c6c1-4a63-a0e6-3ea730c40cdb'
Success: record contains all of the expected fields/values
Success: ./logging.sh passed
++ echo Success: ./logging.sh passed
++ exit 0
running test test-mux.sh
++ set -o nounset
++ set -o pipefail
++ type get_running_pod
++ '[' false == false -o false == false ']'
Skipping -- This test requires both USE_MUX_CLIENT and MUX_ALLOW_EXTERNAL are true.
++ echo 'Skipping -- This test requires both USE_MUX_CLIENT and MUX_ALLOW_EXTERNAL are true.'
++ exit 0
SKIPPING upgrade test for now
running test test-viaq-data-model.sh
++ set -o nounset
++ set -o pipefail
++ type get_running_pod
++ [[ 1 -ne 1 ]]
++ [[ true = \f\a\l\s\e ]]
++ CLUSTER=true
++ ops=-ops
++ INDEX_PREFIX=
++ PROJ_PREFIX=project.
++ ARTIFACT_DIR=/tmp/origin-aggregated-logging/artifacts
++ '[' '!' -d /tmp/origin-aggregated-logging/artifacts ']'
++ get_test_user_token
++ local current_project
+++ oc project -q
++ current_project=logging
++ oc login --username=admin --password=admin
+++ oc whoami -t
++ test_token=2jcwR9RrLfc74_vuEgMdd8YOf8LPg2UOtrIilUyUFmo
+++ oc whoami
++ test_name=admin
++ test_ip=127.0.0.1
++ oc login --username=system:admin
++ oc project logging
++ TEST_DIVIDER=------------------------------------------
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-k81b9
++ remove_test_volume
++ oc get template logging-fluentd-template -o json
++ python -c 'import json, sys; obj = json.loads(sys.stdin.read()); vm = obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"]; obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] = [xx for xx in vm if xx["name"] != "cdmtest"]; vs = obj["objects"][0]["spec"]["template"]["spec"]["volumes"]; obj["objects"][0]["spec"]["template"]["spec"]["volumes"] = [xx for xx in vs if xx["name"] != "cdmtest"]; print json.dumps(obj, indent=2)'
++ oc replace -f -
template "logging-fluentd-template" replaced
+++ mktemp
++ cfg=/tmp/tmp.dJEojPjGvg
++ cat
++ add_test_volume /tmp/tmp.dJEojPjGvg
++ oc get template logging-fluentd-template -o json
++ python -c 'import json, sys; obj = json.loads(sys.stdin.read()); obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"].append({"name": "cdmtest", "mountPath": "/etc/fluent/configs.d/openshift/filter-pre-cdm-test.conf", "readOnly": True}); obj["objects"][0]["spec"]["template"]["spec"]["volumes"].append({"name": "cdmtest", "hostPath": {"path": "/tmp/tmp.dJEojPjGvg"}}); print json.dumps(obj, indent=2)'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ trap cleanup INT TERM EXIT
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-k81b9
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-k81b9
++ '[' -z logging-fluentd-k81b9 -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-k81b9
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
pod for component=fluentd not running yet
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-5k2g9
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-5k2g9 ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-5k2g9
++ keep_fields=method,statusCode,type,@timestamp,req,res
++ write_and_verify_logs test1
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1497028372
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-09 17:12:52.198427611+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 17:12:52.198427611+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-oawpjydu-1-f676p
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
++ '[' -z logging-es-ops-data-master-n5j3wlaj-1-h1fvk ']'
+++ uuidgen
++ local uuid_es=3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
+++ uuidgen
++ local uuid_es_ops=d7c81b66-3aa6-4788-b2ee-eec332440620
++ local expected=1
++ local timeout=300
++ add_test_message 3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-nxzg9
++ oc exec logging-kibana-1-nxzg9 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
++ echo added es message 3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
added es message 3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
++ logger -i -p local6.info -t d7c81b66-3aa6-4788-b2ee-eec332440620 d7c81b66-3aa6-4788-b2ee-eec332440620
added es-ops message d7c81b66-3aa6-4788-b2ee-eec332440620
++ echo added es-ops message d7c81b66-3aa6-4788-b2ee-eec332440620
++ local rc=0
++ espod=logging-es-data-master-oawpjydu-1-f676p
++ myproject=project.logging
++ mymessage=3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1' --connect-timeout 1
+++ get_count_from_json
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1' --connect-timeout 1
+++ get_count_from_json
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
+++ shift
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 292 -le 0 ']'
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
++ espod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
++ myproject=.operations
++ mymessage=d7c81b66-3aa6-4788-b2ee-eec332440620
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk .operations _count systemd.u.SYSLOG_IDENTIFIER d7c81b66-3aa6-4788-b2ee-eec332440620
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:d7c81b66-3aa6-4788-b2ee-eec332440620' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:d7c81b66-3aa6-4788-b2ee-eec332440620'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-n5j3wlaj-1-h1fvk -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:d7c81b66-3aa6-4788-b2ee-eec332440620'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk .operations _count systemd.u.SYSLOG_IDENTIFIER d7c81b66-3aa6-4788-b2ee-eec332440620
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:d7c81b66-3aa6-4788-b2ee-eec332440620' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:d7c81b66-3aa6-4788-b2ee-eec332440620'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-n5j3wlaj-1-h1fvk -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:d7c81b66-3aa6-4788-b2ee-eec332440620'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk .operations _count systemd.u.SYSLOG_IDENTIFIER d7c81b66-3aa6-4788-b2ee-eec332440620
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:d7c81b66-3aa6-4788-b2ee-eec332440620' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:d7c81b66-3aa6-4788-b2ee-eec332440620'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-n5j3wlaj-1-h1fvk -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:d7c81b66-3aa6-4788-b2ee-eec332440620'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk .operations _count systemd.u.SYSLOG_IDENTIFIER d7c81b66-3aa6-4788-b2ee-eec332440620
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:d7c81b66-3aa6-4788-b2ee-eec332440620' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:d7c81b66-3aa6-4788-b2ee-eec332440620'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-n5j3wlaj-1-h1fvk -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:d7c81b66-3aa6-4788-b2ee-eec332440620'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 297 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for d7c81b66-3aa6-4788-b2ee-eec332440620
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for d7c81b66-3aa6-4788-b2ee-eec332440620
++ '[' -n get_logmessage ']'
++ get_logmessage 3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
++ logmessage=3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 d7c81b66-3aa6-4788-b2ee-eec332440620
++ logmessage2=d7c81b66-3aa6-4788-b2ee-eec332440620
+++ date +%s
++ local endtime=1497028390
+++ expr 1497028390 - 1497028372
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 18 seconds at 2017-06-09 17:13:10.069338627+00:00
++ echo END wait_for_fluentd_to_catch_up took 18 seconds at 2017-06-09 17:13:10.069338627+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-nxzg9
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-nxzg9 logging-es project.logging _search message 3c3ea2cd-057e-492c-9553-fe69d8bdc0e1
++ python test-viaq-data-model.py test1
++ oc exec logging-kibana-1-nxzg9 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer 2jcwR9RrLfc74_vuEgMdd8YOf8LPg2UOtrIilUyUFmo' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:3c3ea2cd-057e-492c-9553-fe69d8bdc0e1'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-nxzg9 logging-es-ops .operations _search message d7c81b66-3aa6-4788-b2ee-eec332440620
++ python test-viaq-data-model.py test1
++ oc exec logging-kibana-1-nxzg9 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer 2jcwR9RrLfc74_vuEgMdd8YOf8LPg2UOtrIilUyUFmo' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:d7c81b66-3aa6-4788-b2ee-eec332440620'
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ add_cdm_env_var_val CDM_USE_UNDEFINED '"true"'
+++ mktemp
++ junk=/tmp/tmp.BFpRdr8TJX
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.BFpRdr8TJX'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.BFpRdr8TJX
++ add_cdm_env_var_val CDM_EXTRA_KEEP_FIELDS method,statusCode,type,@timestamp,req,res
+++ mktemp
++ junk=/tmp/tmp.caw8u0Q9p9
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.caw8u0Q9p9'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.caw8u0Q9p9
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-5k2g9
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-5k2g9
++ '[' -z logging-fluentd-5k2g9 -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-5k2g9
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
pod for component=fluentd not running yet
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-t24mn
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-t24mn ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-t24mn
++ write_and_verify_logs test2
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1497028412
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-09 17:13:32.077142397+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 17:13:32.077142397+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-oawpjydu-1-f676p
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
++ '[' -z logging-es-ops-data-master-n5j3wlaj-1-h1fvk ']'
+++ uuidgen
++ local uuid_es=7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
+++ uuidgen
++ local uuid_es_ops=77fd1555-cc16-4efe-8fb3-79df027c1ceb
++ local expected=1
++ local timeout=300
++ add_test_message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-nxzg9
++ oc exec logging-kibana-1-nxzg9 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
added es message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
++ echo added es message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
++ logger -i -p local6.info -t 77fd1555-cc16-4efe-8fb3-79df027c1ceb 77fd1555-cc16-4efe-8fb3-79df027c1ceb
added es-ops message 77fd1555-cc16-4efe-8fb3-79df027c1ceb
++ echo added es-ops message 77fd1555-cc16-4efe-8fb3-79df027c1ceb
++ local rc=0
++ espod=logging-es-data-master-oawpjydu-1-f676p
++ myproject=project.logging
++ mymessage=7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 291 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 290 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 289 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 288 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 288 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
++ espod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
++ myproject=.operations
++ mymessage=77fd1555-cc16-4efe-8fb3-79df027c1ceb
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk .operations _count systemd.u.SYSLOG_IDENTIFIER 77fd1555-cc16-4efe-8fb3-79df027c1ceb
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:77fd1555-cc16-4efe-8fb3-79df027c1ceb' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:77fd1555-cc16-4efe-8fb3-79df027c1ceb'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-n5j3wlaj-1-h1fvk -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:77fd1555-cc16-4efe-8fb3-79df027c1ceb'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 77fd1555-cc16-4efe-8fb3-79df027c1ceb
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 77fd1555-cc16-4efe-8fb3-79df027c1ceb
++ '[' -n get_logmessage ']'
++ get_logmessage 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
++ logmessage=7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 77fd1555-cc16-4efe-8fb3-79df027c1ceb
++ logmessage2=77fd1555-cc16-4efe-8fb3-79df027c1ceb
+++ date +%s
++ local endtime=1497028431
+++ expr 1497028431 - 1497028412
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 19 seconds at 2017-06-09 17:13:51.464556275+00:00
++ echo END wait_for_fluentd_to_catch_up took 19 seconds at 2017-06-09 17:13:51.464556275+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-nxzg9
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-nxzg9 logging-es project.logging _search message 7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a
++ python test-viaq-data-model.py test2
++ oc exec logging-kibana-1-nxzg9 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer 2jcwR9RrLfc74_vuEgMdd8YOf8LPg2UOtrIilUyUFmo' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:7f6d08f0-ebd9-4a4c-99bb-5ea75d27714a'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-nxzg9 logging-es-ops .operations _search message 77fd1555-cc16-4efe-8fb3-79df027c1ceb
++ python test-viaq-data-model.py test2
++ oc exec logging-kibana-1-nxzg9 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer 2jcwR9RrLfc74_vuEgMdd8YOf8LPg2UOtrIilUyUFmo' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:77fd1555-cc16-4efe-8fb3-79df027c1ceb'
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ del_cdm_env_var CDM_EXTRA_KEEP_FIELDS
++ oc get template logging-fluentd-template -o yaml
++ sed '/- name: CDM_EXTRA_KEEP_FIELDS$/,/value:/d'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ add_cdm_env_var_val CDM_EXTRA_KEEP_FIELDS undefined4,undefined5,method,statusCode,type,@timestamp,req,res
+++ mktemp
++ junk=/tmp/tmp.0E8UbktcqM
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.0E8UbktcqM'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.0E8UbktcqM
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-t24mn
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-t24mn
++ '[' -z logging-fluentd-t24mn -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-t24mn
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
pod for component=fluentd not running yet
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-vl809
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-vl809 ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-vl809
++ write_and_verify_logs test3
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1497028452
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-09 17:14:12.036138198+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 17:14:12.036138198+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-oawpjydu-1-f676p
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
++ '[' -z logging-es-ops-data-master-n5j3wlaj-1-h1fvk ']'
+++ uuidgen
++ local uuid_es=bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
+++ uuidgen
++ local uuid_es_ops=30575da0-db6a-42d9-bda5-9f734215882d
++ local expected=1
++ local timeout=300
++ add_test_message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-nxzg9
++ oc exec logging-kibana-1-nxzg9 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
added es message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
++ echo added es message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
++ logger -i -p local6.info -t 30575da0-db6a-42d9-bda5-9f734215882d 30575da0-db6a-42d9-bda5-9f734215882d
added es-ops message 30575da0-db6a-42d9-bda5-9f734215882d
++ echo added es-ops message 30575da0-db6a-42d9-bda5-9f734215882d
++ local rc=0
++ espod=logging-es-data-master-oawpjydu-1-f676p
++ myproject=project.logging
++ mymessage=bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 291 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 290 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 289 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 288 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
good - wait_for_fluentd_to_catch_up: found 1 record project logging for bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 288 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
++ espod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
++ myproject=.operations
++ mymessage=30575da0-db6a-42d9-bda5-9f734215882d
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk .operations _count systemd.u.SYSLOG_IDENTIFIER 30575da0-db6a-42d9-bda5-9f734215882d
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:30575da0-db6a-42d9-bda5-9f734215882d' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:30575da0-db6a-42d9-bda5-9f734215882d'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-n5j3wlaj-1-h1fvk -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:30575da0-db6a-42d9-bda5-9f734215882d'
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 30575da0-db6a-42d9-bda5-9f734215882d
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 30575da0-db6a-42d9-bda5-9f734215882d
++ '[' -n get_logmessage ']'
++ get_logmessage bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
++ logmessage=bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 30575da0-db6a-42d9-bda5-9f734215882d
++ logmessage2=30575da0-db6a-42d9-bda5-9f734215882d
+++ date +%s
++ local endtime=1497028471
+++ expr 1497028471 - 1497028452
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 19 seconds at 2017-06-09 17:14:31.268773079+00:00
++ echo END wait_for_fluentd_to_catch_up took 19 seconds at 2017-06-09 17:14:31.268773079+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-nxzg9
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-nxzg9 logging-es project.logging _search message bc9bb09f-4099-46bd-bdf6-1b10c5e869c0
++ python test-viaq-data-model.py test3
++ oc exec logging-kibana-1-nxzg9 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer 2jcwR9RrLfc74_vuEgMdd8YOf8LPg2UOtrIilUyUFmo' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:bc9bb09f-4099-46bd-bdf6-1b10c5e869c0'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-nxzg9 logging-es-ops .operations _search message 30575da0-db6a-42d9-bda5-9f734215882d
++ oc exec logging-kibana-1-nxzg9 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer 2jcwR9RrLfc74_vuEgMdd8YOf8LPg2UOtrIilUyUFmo' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:30575da0-db6a-42d9-bda5-9f734215882d'
++ python test-viaq-data-model.py test3
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ add_cdm_env_var_val CDM_UNDEFINED_NAME myname
+++ mktemp
++ junk=/tmp/tmp.HbquYK7BqY
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.HbquYK7BqY'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.HbquYK7BqY
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-vl809
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-vl809
++ '[' -z logging-fluentd-vl809 -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-vl809
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
pod for component=fluentd not running yet
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-9fqx3
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-9fqx3 ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-9fqx3
++ write_and_verify_logs test4
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1497028488
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-09 17:14:48.366738855+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 17:14:48.366738855+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-oawpjydu-1-f676p
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
++ '[' -z logging-es-ops-data-master-n5j3wlaj-1-h1fvk ']'
+++ uuidgen
++ local uuid_es=9da5cfd5-c8c9-47cf-b514-7d674b14bca5
+++ uuidgen
++ local uuid_es_ops=80709a4c-cca4-4d86-8034-96615e07a50b
++ local expected=1
++ local timeout=300
++ add_test_message 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-nxzg9
++ oc exec logging-kibana-1-nxzg9 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/9da5cfd5-c8c9-47cf-b514-7d674b14bca5
added es message 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
++ echo added es message 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
++ logger -i -p local6.info -t 80709a4c-cca4-4d86-8034-96615e07a50b 80709a4c-cca4-4d86-8034-96615e07a50b
added es-ops message 80709a4c-cca4-4d86-8034-96615e07a50b
++ echo added es-ops message 80709a4c-cca4-4d86-8034-96615e07a50b
++ local rc=0
++ espod=logging-es-data-master-oawpjydu-1-f676p
++ myproject=project.logging
++ mymessage=9da5cfd5-c8c9-47cf-b514-7d674b14bca5
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
+++ shift
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 291 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 290 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 289 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 289 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
++ espod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
++ myproject=.operations
++ mymessage=80709a4c-cca4-4d86-8034-96615e07a50b
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk .operations _count systemd.u.SYSLOG_IDENTIFIER 80709a4c-cca4-4d86-8034-96615e07a50b
+++ curl_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:80709a4c-cca4-4d86-8034-96615e07a50b' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:80709a4c-cca4-4d86-8034-96615e07a50b'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-n5j3wlaj-1-h1fvk -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:80709a4c-cca4-4d86-8034-96615e07a50b'
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk .operations _count systemd.u.SYSLOG_IDENTIFIER 80709a4c-cca4-4d86-8034-96615e07a50b
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:80709a4c-cca4-4d86-8034-96615e07a50b' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:80709a4c-cca4-4d86-8034-96615e07a50b'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-n5j3wlaj-1-h1fvk -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:80709a4c-cca4-4d86-8034-96615e07a50b'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk .operations _count systemd.u.SYSLOG_IDENTIFIER 80709a4c-cca4-4d86-8034-96615e07a50b
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:80709a4c-cca4-4d86-8034-96615e07a50b' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:80709a4c-cca4-4d86-8034-96615e07a50b'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-n5j3wlaj-1-h1fvk -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:80709a4c-cca4-4d86-8034-96615e07a50b'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk .operations _count systemd.u.SYSLOG_IDENTIFIER 80709a4c-cca4-4d86-8034-96615e07a50b
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:80709a4c-cca4-4d86-8034-96615e07a50b' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:80709a4c-cca4-4d86-8034-96615e07a50b'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-n5j3wlaj-1-h1fvk -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:80709a4c-cca4-4d86-8034-96615e07a50b'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 297 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 80709a4c-cca4-4d86-8034-96615e07a50b
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 80709a4c-cca4-4d86-8034-96615e07a50b
++ '[' -n get_logmessage ']'
++ get_logmessage 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
++ logmessage=9da5cfd5-c8c9-47cf-b514-7d674b14bca5
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 80709a4c-cca4-4d86-8034-96615e07a50b
++ logmessage2=80709a4c-cca4-4d86-8034-96615e07a50b
+++ date +%s
++ local endtime=1497028510
+++ expr 1497028510 - 1497028488
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 22 seconds at 2017-06-09 17:15:10.504487260+00:00
++ echo END wait_for_fluentd_to_catch_up took 22 seconds at 2017-06-09 17:15:10.504487260+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-nxzg9
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-nxzg9 logging-es project.logging _search message 9da5cfd5-c8c9-47cf-b514-7d674b14bca5
++ python test-viaq-data-model.py test4
++ oc exec logging-kibana-1-nxzg9 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer 2jcwR9RrLfc74_vuEgMdd8YOf8LPg2UOtrIilUyUFmo' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:9da5cfd5-c8c9-47cf-b514-7d674b14bca5'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-nxzg9 logging-es-ops .operations _search message 80709a4c-cca4-4d86-8034-96615e07a50b
++ python test-viaq-data-model.py test4
++ oc exec logging-kibana-1-nxzg9 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer 2jcwR9RrLfc74_vuEgMdd8YOf8LPg2UOtrIilUyUFmo' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:80709a4c-cca4-4d86-8034-96615e07a50b'
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ del_cdm_env_var CDM_EXTRA_KEEP_FIELDS
++ oc get template logging-fluentd-template -o yaml
++ oc replace -f -
++ sed '/- name: CDM_EXTRA_KEEP_FIELDS$/,/value:/d'
template "logging-fluentd-template" replaced
++ add_cdm_env_var_val CDM_EXTRA_KEEP_FIELDS undefined4,undefined5,empty1,undefined3,method,statusCode,type,@timestamp,req,res
+++ mktemp
++ junk=/tmp/tmp.PCT7E5Mxaf
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.PCT7E5Mxaf'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.PCT7E5Mxaf
++ add_cdm_env_var_val CDM_KEEP_EMPTY_FIELDS undefined4,undefined5,empty1,undefined3
+++ mktemp
++ junk=/tmp/tmp.pfglr1MDev
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.pfglr1MDev'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.pfglr1MDev
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-9fqx3
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-9fqx3
++ '[' -z logging-fluentd-9fqx3 -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-9fqx3
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
pod for component=fluentd not running yet
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-hlw91
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-hlw91 ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-hlw91
++ write_and_verify_logs test5 allow_empty
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1497028528
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-09 17:15:28.617351497+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 17:15:28.617351497+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-oawpjydu-1-f676p
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
++ '[' -z logging-es-ops-data-master-n5j3wlaj-1-h1fvk ']'
+++ uuidgen
++ local uuid_es=f629100c-8636-4d6b-91fa-ba31e64c5afe
+++ uuidgen
++ local uuid_es_ops=ba7aeca8-b88a-4359-80d9-5b3347e0c1fd
++ local expected=1
++ local timeout=300
++ add_test_message f629100c-8636-4d6b-91fa-ba31e64c5afe
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-nxzg9
++ oc exec logging-kibana-1-nxzg9 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/f629100c-8636-4d6b-91fa-ba31e64c5afe
added es message f629100c-8636-4d6b-91fa-ba31e64c5afe
++ echo added es message f629100c-8636-4d6b-91fa-ba31e64c5afe
++ logger -i -p local6.info -t ba7aeca8-b88a-4359-80d9-5b3347e0c1fd ba7aeca8-b88a-4359-80d9-5b3347e0c1fd
added es-ops message ba7aeca8-b88a-4359-80d9-5b3347e0c1fd
++ echo added es-ops message ba7aeca8-b88a-4359-80d9-5b3347e0c1fd
++ local rc=0
++ espod=logging-es-data-master-oawpjydu-1-f676p
++ myproject=project.logging
++ mymessage=f629100c-8636-4d6b-91fa-ba31e64c5afe
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message f629100c-8636-4d6b-91fa-ba31e64c5afe
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message f629100c-8636-4d6b-91fa-ba31e64c5afe
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message f629100c-8636-4d6b-91fa-ba31e64c5afe
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message f629100c-8636-4d6b-91fa-ba31e64c5afe
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message f629100c-8636-4d6b-91fa-ba31e64c5afe
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message f629100c-8636-4d6b-91fa-ba31e64c5afe
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message f629100c-8636-4d6b-91fa-ba31e64c5afe
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message f629100c-8636-4d6b-91fa-ba31e64c5afe
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message f629100c-8636-4d6b-91fa-ba31e64c5afe
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 291 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message f629100c-8636-4d6b-91fa-ba31e64c5afe
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 290 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message f629100c-8636-4d6b-91fa-ba31e64c5afe
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ local 'endpoint=/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
+++ shift
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 289 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-oawpjydu-1-f676p project.logging _count message f629100c-8636-4d6b-91fa-ba31e64c5afe
+++ get_count_from_json
+++ curl_es logging-es-data-master-oawpjydu-1-f676p '/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe' --connect-timeout 1
+++ local pod=logging-es-data-master-oawpjydu-1-f676p
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-oawpjydu-1-f676p -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 289 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for f629100c-8636-4d6b-91fa-ba31e64c5afe
good - wait_for_fluentd_to_catch_up: found 1 record project logging for f629100c-8636-4d6b-91fa-ba31e64c5afe
++ espod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
++ myproject=.operations
++ mymessage=ba7aeca8-b88a-4359-80d9-5b3347e0c1fd
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk .operations _count systemd.u.SYSLOG_IDENTIFIER ba7aeca8-b88a-4359-80d9-5b3347e0c1fd
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-n5j3wlaj-1-h1fvk '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:ba7aeca8-b88a-4359-80d9-5b3347e0c1fd' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-n5j3wlaj-1-h1fvk
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:ba7aeca8-b88a-4359-80d9-5b3347e0c1fd'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-n5j3wlaj-1-h1fvk -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:ba7aeca8-b88a-4359-80d9-5b3347e0c1fd'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for ba7aeca8-b88a-4359-80d9-5b3347e0c1fd
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for ba7aeca8-b88a-4359-80d9-5b3347e0c1fd
++ '[' -n get_logmessage ']'
++ get_logmessage f629100c-8636-4d6b-91fa-ba31e64c5afe
++ logmessage=f629100c-8636-4d6b-91fa-ba31e64c5afe
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 ba7aeca8-b88a-4359-80d9-5b3347e0c1fd
++ logmessage2=ba7aeca8-b88a-4359-80d9-5b3347e0c1fd
+++ date +%s
++ local endtime=1497028546
+++ expr 1497028546 - 1497028528
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 18 seconds at 2017-06-09 17:15:46.726240177+00:00
++ echo END wait_for_fluentd_to_catch_up took 18 seconds at 2017-06-09 17:15:46.726240177+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-nxzg9
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-nxzg9 logging-es project.logging _search message f629100c-8636-4d6b-91fa-ba31e64c5afe
++ python test-viaq-data-model.py test5 allow_empty
++ oc exec logging-kibana-1-nxzg9 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer 2jcwR9RrLfc74_vuEgMdd8YOf8LPg2UOtrIilUyUFmo' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:f629100c-8636-4d6b-91fa-ba31e64c5afe'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-nxzg9 logging-es-ops .operations _search message ba7aeca8-b88a-4359-80d9-5b3347e0c1fd
++ python test-viaq-data-model.py test5 allow_empty
++ oc exec logging-kibana-1-nxzg9 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer 2jcwR9RrLfc74_vuEgMdd8YOf8LPg2UOtrIilUyUFmo' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:ba7aeca8-b88a-4359-80d9-5b3347e0c1fd'
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ cleanup
++ remove_test_volume
++ oc get template logging-fluentd-template -o json
++ python -c 'import json, sys; obj = json.loads(sys.stdin.read()); vm = obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"]; obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] = [xx for xx in vm if xx["name"] != "cdmtest"]; vs = obj["objects"][0]["spec"]["template"]["spec"]["volumes"]; obj["objects"][0]["spec"]["template"]["spec"]["volumes"] = [xx for xx in vs if xx["name"] != "cdmtest"]; print json.dumps(obj, indent=2)'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ remove_cdm_env
++ oc get template logging-fluentd-template -o yaml
++ sed '/- name: CDM_/,/value:/d'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.dJEojPjGvg
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-hlw91
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-hlw91
++ '[' -z logging-fluentd-hlw91 -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-hlw91
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
pod for component=fluentd not running yet
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
SKIPPING reinstall test for now
++ curpod=logging-fluentd-n4khx
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-n4khx ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
/data/src/github.com/openshift/origin-aggregated-logging/hack/lib/log/system.sh: line 31:  4015 Terminated              sar -A -o "${binary_logfile}" 1 86400 > /dev/null 2> "${stderr_logfile}"  (wd: /data/src/github.com/openshift/origin-aggregated-logging)
[INFO] [CLEANUP] Beginning cleanup routines...
[INFO] [CLEANUP] Dumping cluster events to /tmp/origin-aggregated-logging/artifacts/events.txt
[INFO] [CLEANUP] Dumping etcd contents to /tmp/origin-aggregated-logging/artifacts/etcd
[WARNING] No compiled `etcdhelper` binary was found. Attempting to build one using:
[WARNING]   $ hack/build-go.sh tools/etcdhelper
++ Building go targets for linux/amd64: tools/etcdhelper
/data/src/github.com/openshift/origin-aggregated-logging/../origin/hack/build-go.sh took 185 seconds
[INFO] [CLEANUP] Dumping container logs to /tmp/origin-aggregated-logging/logs/containers
[INFO] [CLEANUP] Truncating log files over 200M
[INFO] [CLEANUP] Stopping docker containers
[INFO] [CLEANUP] Removing docker containers
Error response from daemon: You cannot remove a running container bea3bf43f8bc3141ef3031dde833cd1e6f708dcabe1664718237b3f6ef4e104c. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 9952c570c4f06b79cadb37bf178af60e7df3acef1b3f236ca1a25f2fc0df2332. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 5f8edc5f3918e616e4861c9c0d55ef7769de959f5bd7453d9eff91525e018974. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container fc510ffd2142604e7563b613e3a3cd4d16cf14ff49733278c3c2c4137c61c868. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 384993f64490d228889f42a2cb840051096ee943430c7867a87feee8a1532f4e. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 82a4773aa4285b0a475bb5a4ba26b36521c4d36a40e9f6424aea0baeeb55c135. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container a966aa20a60f290958428a247fba324c1b74f8632ea33c1af890d2440b013946. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 38f142c997dca472c96af732b2cbc678be7999106270825a7f43304afcb75249. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 7c2f49787b33a07f616c2688432dd511839168c5d0d86130438c06afb738b53c. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 33ccd98cbc765603140f7a718d2aee1de1c1184583972a31ef5409b51f8fa218. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container aba6562b23d64c26a528d22249487073156a75bee337020f46f69b05e8ae436f. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container c086cc8d8315ce225980791113c9840d4e796c40cc5a724cf94f0a5f5ff43e11. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container b64c8da1d82a1e394f4200acc13db108c86d46313476f325f7ebf78fd59aca62. Stop the container before attempting removal or use -f
[INFO] [CLEANUP] Killing child processes
[INFO] [CLEANUP] Pruning etcd data directory
[INFO] /data/src/github.com/openshift/origin-aggregated-logging/logging.sh exited with code 0 after 01h 12m 12s
/data/src/github.com/openshift/origin-aggregated-logging
+ popd
+ set +o xtrace
########## FINISHED STAGE: SUCCESS: RUN LOGGING TESTS [01h 12m 13s] ##########
[PostBuildScript] - Execution post build scripts.
[workspace] $ /bin/bash /tmp/hudson8952776583815244279.sh
########## STARTING STAGE: DOWNLOAD ARTIFACTS FROM THE REMOTE HOST ##########
+ [[ -s /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/activate ]]
+ source /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/activate
++ export VIRTUAL_ENV=/var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890
++ VIRTUAL_ENV=/var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890
++ export PATH=/var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/bin:/sbin:/usr/sbin:/bin:/usr/bin
++ PATH=/var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/bin:/sbin:/usr/sbin:/bin:/usr/bin
++ unset PYTHON_HOME
++ export OCT_CONFIG_HOME=/var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/.config
++ OCT_CONFIG_HOME=/var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/.config
++ pwd
+ ARTIFACT_DIR=/var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts
+ rm -rf /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts
+ mkdir /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts
+ ssh -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel sudo stat /data/src/github.com/openshift/origin/_output/scripts
  File: ‘/data/src/github.com/openshift/origin/_output/scripts’
  Size: 59        	Blocks: 0          IO Block: 4096   directory
Device: ca02h/51714d	Inode: 51274771    Links: 5
Access: (2755/drwxr-sr-x)  Uid: ( 1002/  origin)   Gid: ( 1004/origin-git)
Context: unconfined_u:object_r:default_t:s0
Access: 2017-06-09 11:03:44.022596023 -0400
Modify: 2017-06-09 11:26:57.925513166 -0400
Change: 2017-06-09 11:26:57.925513166 -0400
 Birth: -
+ ssh -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel sudo chmod -R o+rX /data/src/github.com/openshift/origin/_output/scripts
+ scp -r -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel:/data/src/github.com/openshift/origin/_output/scripts /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts
+ ssh -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel sudo stat /data/src/github.com/openshift/origin-aggregated-logging/_output/scripts
  File: ‘/data/src/github.com/openshift/origin-aggregated-logging/_output/scripts’
  Size: 38        	Blocks: 0          IO Block: 4096   directory
Device: ca02h/51714d	Inode: 76023304    Links: 3
Access: (2755/drwxr-sr-x)  Uid: ( 1002/  origin)   Gid: ( 1004/origin-git)
Context: unconfined_u:object_r:default_t:s0
Access: 2017-06-09 12:10:50.523539500 -0400
Modify: 2017-06-09 12:08:26.482559971 -0400
Change: 2017-06-09 12:08:26.482559971 -0400
 Birth: -
+ ssh -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel sudo chmod -R o+rX /data/src/github.com/openshift/origin-aggregated-logging/_output/scripts
+ scp -r -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel:/data/src/github.com/openshift/origin-aggregated-logging/_output/scripts /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts
+ tree /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts
/var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts
└── scripts
    ├── build-base-images
    │   ├── artifacts
    │   ├── logs
    │   └── openshift.local.home
    ├── build-images
    │   ├── artifacts
    │   ├── logs
    │   └── openshift.local.home
    ├── env
    │   ├── artifacts
    │   ├── logs
    │   │   └── scripts.log
    │   └── openshift.local.home
    └── origin-aggregated-logging
        └── openshift.local.home

15 directories, 1 file
+ set +o xtrace
########## FINISHED STAGE: SUCCESS: DOWNLOAD ARTIFACTS FROM THE REMOTE HOST [00h 00m 02s] ##########
[workspace] $ /bin/bash /tmp/hudson1004998142437003659.sh
########## STARTING STAGE: GENERATE ARTIFACTS FROM THE REMOTE HOST ##########
+ [[ -s /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/activate ]]
+ source /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/activate
++ export VIRTUAL_ENV=/var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890
++ VIRTUAL_ENV=/var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890
++ export PATH=/var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/bin:/sbin:/usr/sbin:/bin:/usr/bin
++ PATH=/var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/bin:/sbin:/usr/sbin:/bin:/usr/bin
++ unset PYTHON_HOME
++ export OCT_CONFIG_HOME=/var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/.config
++ OCT_CONFIG_HOME=/var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/.config
++ pwd
+ ARTIFACT_DIR=/var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts/generated
+ rm -rf /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts/generated
+ mkdir /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts/generated
+ ssh -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel 'sudo docker version && sudo docker info && sudo docker images && sudo docker ps -a 2>&1'
+ ssh -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel 'sudo df -h && sudo vgs && sudo lvs 2>&1'
+ ssh -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel 'sudo yum list installed 2>&1'
+ ssh -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel 'sudo ausearch -m avc 2>&1'
+ true
+ ssh -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel 'sudo journalctl _PID=1 --no-pager --all --lines=all 2>&1'
+ tree /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts/generated
/var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts/generated
├── avc_denials.log
├── docker.info
├── filesystem.info
├── installed_packages.log
└── pid1.journal

0 directories, 5 files
+ set +o xtrace
########## FINISHED STAGE: SUCCESS: GENERATE ARTIFACTS FROM THE REMOTE HOST [00h 00m 09s] ##########
[workspace] $ /bin/bash /tmp/hudson5433685167769348358.sh
########## STARTING STAGE: FETCH SYSTEMD JOURNALS FROM THE REMOTE HOST ##########
+ [[ -s /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/activate ]]
+ source /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/activate
++ export VIRTUAL_ENV=/var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890
++ VIRTUAL_ENV=/var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890
++ export PATH=/var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/bin:/sbin:/usr/sbin:/bin:/usr/bin
++ PATH=/var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/bin:/sbin:/usr/sbin:/bin:/usr/bin
++ unset PYTHON_HOME
++ export OCT_CONFIG_HOME=/var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/.config
++ OCT_CONFIG_HOME=/var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/.config
++ pwd
+ ARTIFACT_DIR=/var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts/journals
+ rm -rf /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts/journals
+ mkdir /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts/journals
+ ssh -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel sudo journalctl --unit docker.service --no-pager --all --lines=all
+ ssh -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel sudo journalctl --unit origin-master.service --no-pager --all --lines=all
+ ssh -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel sudo journalctl --unit origin-node.service --no-pager --all --lines=all
+ ssh -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel sudo journalctl --unit openvswitch.service --no-pager --all --lines=all
+ ssh -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel sudo journalctl --unit ovs-vswitchd.service --no-pager --all --lines=all
+ ssh -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel sudo journalctl --unit ovsdb-server.service --no-pager --all --lines=all
+ ssh -F ./.config/origin-ci-tool/inventory/.ssh_config openshiftdevel sudo journalctl --unit etcd.service --no-pager --all --lines=all
+ tree /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts/journals
/var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/artifacts/journals
├── docker.service
├── etcd.service
├── openvswitch.service
├── origin-master.service
├── origin-node.service
├── ovsdb-server.service
└── ovs-vswitchd.service

0 directories, 7 files
+ set +o xtrace
########## FINISHED STAGE: SUCCESS: FETCH SYSTEMD JOURNALS FROM THE REMOTE HOST [00h 00m 03s] ##########
[workspace] $ /bin/bash /tmp/hudson9156375445730815424.sh
########## STARTING STAGE: DEPROVISION CLOUD RESOURCES ##########
+ [[ -s /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/activate ]]
+ source /var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/activate
++ export VIRTUAL_ENV=/var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890
++ VIRTUAL_ENV=/var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890
++ export PATH=/var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/bin:/sbin:/usr/sbin:/bin:/usr/bin
++ PATH=/var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/bin:/sbin:/usr/sbin:/bin:/usr/bin
++ unset PYTHON_HOME
++ export OCT_CONFIG_HOME=/var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/.config
++ OCT_CONFIG_HOME=/var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/.config
+ oct deprovision

PLAYBOOK: main.yml *************************************************************
4 plays in /var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/lib/python2.7/site-packages/oct/ansible/oct/playbooks/deprovision/main.yml

PLAY [ensure we have the parameters necessary to deprovision virtual hosts] ****

TASK [ensure all required variables are set] ***********************************
task path: /var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/lib/python2.7/site-packages/oct/ansible/oct/playbooks/deprovision/main.yml:9
skipping: [localhost] => (item=origin_ci_inventory_dir)  => {
    "changed": false, 
    "generated_timestamp": "2017-06-09 13:20:46.512021", 
    "item": "origin_ci_inventory_dir", 
    "skip_reason": "Conditional check failed", 
    "skipped": true
}
skipping: [localhost] => (item=origin_ci_aws_region)  => {
    "changed": false, 
    "generated_timestamp": "2017-06-09 13:20:46.517632", 
    "item": "origin_ci_aws_region", 
    "skip_reason": "Conditional check failed", 
    "skipped": true
}

PLAY [deprovision virtual hosts in EC2] ****************************************

TASK [Gathering Facts] *********************************************************
ok: [localhost]

TASK [deprovision a virtual EC2 host] ******************************************
task path: /var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/lib/python2.7/site-packages/oct/ansible/oct/playbooks/deprovision/main.yml:28
included: /var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/lib/python2.7/site-packages/oct/ansible/oct/playbooks/deprovision/roles/aws-down/tasks/main.yml for localhost

TASK [update the SSH configuration to remove AWS EC2 specifics] ****************
task path: /var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/lib/python2.7/site-packages/oct/ansible/oct/playbooks/deprovision/roles/aws-down/tasks/main.yml:2
ok: [localhost] => {
    "changed": false, 
    "generated_timestamp": "2017-06-09 13:20:47.388315", 
    "msg": ""
}

TASK [rename EC2 instance for termination reaper] ******************************
task path: /var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/lib/python2.7/site-packages/oct/ansible/oct/playbooks/deprovision/roles/aws-down/tasks/main.yml:8
changed: [localhost] => {
    "changed": true, 
    "generated_timestamp": "2017-06-09 13:20:48.024249", 
    "msg": "Tags {'Name': 'terminate'} created for resource i-0c2f17a3c842910cc."
}

TASK [tear down the EC2 instance] **********************************************
task path: /var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/lib/python2.7/site-packages/oct/ansible/oct/playbooks/deprovision/roles/aws-down/tasks/main.yml:15
changed: [localhost] => {
    "changed": true, 
    "generated_timestamp": "2017-06-09 13:20:48.972807", 
    "instance_ids": [
        "i-0c2f17a3c842910cc"
    ], 
    "instances": [
        {
            "ami_launch_index": "0", 
            "architecture": "x86_64", 
            "block_device_mapping": {
                "/dev/sda1": {
                    "delete_on_termination": true, 
                    "status": "attached", 
                    "volume_id": "vol-0c35e37cab2fd5b4f"
                }, 
                "/dev/sdb": {
                    "delete_on_termination": true, 
                    "status": "attached", 
                    "volume_id": "vol-05555f2f10ac24a54"
                }
            }, 
            "dns_name": "ec2-34-207-254-240.compute-1.amazonaws.com", 
            "ebs_optimized": false, 
            "groups": {
                "sg-7e73221a": "default"
            }, 
            "hypervisor": "xen", 
            "id": "i-0c2f17a3c842910cc", 
            "image_id": "ami-2494ca32", 
            "instance_type": "m4.xlarge", 
            "kernel": null, 
            "key_name": "libra", 
            "launch_time": "2017-06-09T16:03:07.000Z", 
            "placement": "us-east-1d", 
            "private_dns_name": "ip-172-18-7-3.ec2.internal", 
            "private_ip": "172.18.7.3", 
            "public_dns_name": "ec2-34-207-254-240.compute-1.amazonaws.com", 
            "public_ip": "34.207.254.240", 
            "ramdisk": null, 
            "region": "us-east-1", 
            "root_device_name": "/dev/sda1", 
            "root_device_type": "ebs", 
            "state": "running", 
            "state_code": 16, 
            "tags": {
                "Name": "terminate", 
                "openshift_etcd": "", 
                "openshift_master": "", 
                "openshift_node": ""
            }, 
            "tenancy": "default", 
            "virtualization_type": "hvm"
        }
    ], 
    "tagged_instances": []
}

TASK [remove the serialized host variables] ************************************
task path: /var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/lib/python2.7/site-packages/oct/ansible/oct/playbooks/deprovision/roles/aws-down/tasks/main.yml:21
changed: [localhost] => {
    "changed": true, 
    "generated_timestamp": "2017-06-09 13:20:49.235977", 
    "path": "/var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/.config/origin-ci-tool/inventory/host_vars/172.18.7.3.yml", 
    "state": "absent"
}

PLAY [deprovision virtual hosts locally manged by Vagrant] *********************

TASK [Gathering Facts] *********************************************************
ok: [localhost]

PLAY [clean up local configuration for deprovisioned instances] ****************

TASK [remove inventory configuration directory] ********************************
task path: /var/lib/jenkins/origin-ci-tool/9aea3b4f81e266b026e21975a3a6a5a1cfddd890/lib/python2.7/site-packages/oct/ansible/oct/playbooks/deprovision/main.yml:61
changed: [localhost] => {
    "changed": true, 
    "generated_timestamp": "2017-06-09 13:20:49.749412", 
    "path": "/var/lib/jenkins/jobs/test_branch_origin_aggregated_logging_prior/workspace/.config/origin-ci-tool/inventory", 
    "state": "absent"
}

PLAY RECAP *********************************************************************
localhost                  : ok=8    changed=4    unreachable=0    failed=0   

+ set +o xtrace
########## FINISHED STAGE: SUCCESS: DEPROVISION CLOUD RESOURCES [00h 00m 04s] ##########
Archiving artifacts
Recording test results
Finished: SUCCESS