SuccessConsole Output

Skipping 607 KB.. Full Log
od=logging-es-ops-data-master-gw0u0psf-1-j2gcm
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:7cedd6e1-af4e-43de-b614-7789c121cc56'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-gw0u0psf-1-j2gcm -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:7cedd6e1-af4e-43de-b614-7789c121cc56'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-gw0u0psf-1-j2gcm .operations _count systemd.u.SYSLOG_IDENTIFIER 7cedd6e1-af4e-43de-b614-7789c121cc56
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-gw0u0psf-1-j2gcm '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:7cedd6e1-af4e-43de-b614-7789c121cc56' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:7cedd6e1-af4e-43de-b614-7789c121cc56'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-gw0u0psf-1-j2gcm -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:7cedd6e1-af4e-43de-b614-7789c121cc56'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 297 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 7cedd6e1-af4e-43de-b614-7789c121cc56
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 7cedd6e1-af4e-43de-b614-7789c121cc56
++ '[' -n '' ']'
++ '[' -n '' ']'
+++ date +%s
++ local endtime=1496970729
+++ expr 1496970729 - 1496970644
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 85 seconds at 2017-06-09 01:12:09.011300280+00:00
++ echo END wait_for_fluentd_to_catch_up took 85 seconds at 2017-06-09 01:12:09.011300280+00:00
++ return 0
++ return 0
++ cleanup
++ cleanup_forward
++ oc label node --all logging-infra-fluentd-
node "172.18.10.53" labeled
++ wait_for_pod_ACTION stop logging-fluentd-5fhbg
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-5fhbg
++ '[' -z logging-fluentd-5fhbg -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-5fhbg
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc delete daemonset/logging-forward-fluentd
daemonset "logging-forward-fluentd" deleted
+++ oc get configmap/logging-fluentd -o yaml
+++ grep '<match \*\*>'
++ '[' -n '      <match **>' ']'
++ oc get configmap/logging-fluentd -o yaml
++ sed -e '/<match \*\*>/ d' -e '/@include configs\.d\/user\/secure-forward\.conf/ d' -e '/<\/match>/ d'
++ oc replace -f -
configmap "logging-fluentd" replaced
++ oc patch configmap/logging-fluentd --type=json --patch '[{ "op": "replace", "path": "/data/secure-forward.conf", "value": "\
# @type secure_forward\n\
# self_hostname forwarding-${HOSTNAME}\n\
# shared_key aggregated_logging_ci_testing\n\
#  secure no\n\
#  <server>\n\
#   host ${FLUENTD_FORWARD}\n\
#   port 24284\n\
#  </server>"}]'
configmap "logging-fluentd" patched
++ oc label node --all logging-infra-fluentd=true
node "172.18.10.53" labeled
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
pod for component=fluentd not running yet
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-v25br
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-v25br ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-v25br
++ oc get events -o yaml
++ write_and_verify_logs 1
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up '' ''
+++ date +%s
++ local starttime=1496970744
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-09 01:12:24.606955944+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 01:12:24.606955944+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
++ '[' -z logging-es-ops-data-master-gw0u0psf-1-j2gcm ']'
+++ uuidgen
++ local uuid_es=345813f9-38cc-4f9c-bf56-9df6ac91c8e9
+++ uuidgen
++ local uuid_es_ops=48ee6f6d-a613-4423-9045-37d1515d99fc
++ local expected=1
++ local timeout=300
++ add_test_message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-1fms4
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/345813f9-38cc-4f9c-bf56-9df6ac91c8e9
added es message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
++ echo added es message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
++ logger -i -p local6.info -t 48ee6f6d-a613-4423-9045-37d1515d99fc 48ee6f6d-a613-4423-9045-37d1515d99fc
added es-ops message 48ee6f6d-a613-4423-9045-37d1515d99fc
++ echo added es-ops message 48ee6f6d-a613-4423-9045-37d1515d99fc
++ local rc=0
++ espod=logging-es-data-master-f10vs1am-1-wm1tr
++ myproject=project.logging
++ mymessage=345813f9-38cc-4f9c-bf56-9df6ac91c8e9
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
+++ shift
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 291 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 290 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 289 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ get_count_from_json
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 288 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 287 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:345813f9-38cc-4f9c-bf56-9df6ac91c8e9'
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 287 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 345813f9-38cc-4f9c-bf56-9df6ac91c8e9
++ espod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
++ myproject=.operations
++ mymessage=48ee6f6d-a613-4423-9045-37d1515d99fc
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-gw0u0psf-1-j2gcm .operations _count systemd.u.SYSLOG_IDENTIFIER 48ee6f6d-a613-4423-9045-37d1515d99fc
+++ curl_es logging-es-ops-data-master-gw0u0psf-1-j2gcm '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:48ee6f6d-a613-4423-9045-37d1515d99fc' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:48ee6f6d-a613-4423-9045-37d1515d99fc'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-gw0u0psf-1-j2gcm -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:48ee6f6d-a613-4423-9045-37d1515d99fc'
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 48ee6f6d-a613-4423-9045-37d1515d99fc
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 48ee6f6d-a613-4423-9045-37d1515d99fc
++ '[' -n '' ']'
++ '[' -n '' ']'
+++ date +%s
++ local endtime=1496970764
+++ expr 1496970764 - 1496970744
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 20 seconds at 2017-06-09 01:12:44.641416811+00:00
++ echo END wait_for_fluentd_to_catch_up took 20 seconds at 2017-06-09 01:12:44.641416811+00:00
++ return 0
++ return 0
++ cleanup
++ cleanup_forward
++ oc label node --all logging-infra-fluentd-
node "172.18.10.53" labeled
++ wait_for_pod_ACTION stop logging-fluentd-v25br
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-v25br
++ '[' -z logging-fluentd-v25br -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-v25br
++ '[' -n 1 ']'
pod logging-fluentd-v25br still running
++ echo pod logging-fluentd-v25br still running
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' stop = start ']'
++ '[' 110 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-v25br
++ '[' stop = start ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
++ oc delete daemonset/logging-forward-fluentd
Error from server (NotFound): daemonsets.extensions "logging-forward-fluentd" not found
++ :
+++ oc get configmap/logging-fluentd -o yaml
+++ grep '<match \*\*>'
++ '[' -n '' ']'
++ oc patch configmap/logging-fluentd --type=json --patch '[{ "op": "replace", "path": "/data/secure-forward.conf", "value": "\
# @type secure_forward\n\
# self_hostname forwarding-${HOSTNAME}\n\
# shared_key aggregated_logging_ci_testing\n\
#  secure no\n\
#  <server>\n\
#   host ${FLUENTD_FORWARD}\n\
#   port 24284\n\
#  </server>"}]'
configmap "logging-fluentd" not patched
++ oc label node --all logging-infra-fluentd=true
node "172.18.10.53" labeled
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-fnpvk
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-fnpvk ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-fnpvk
++ oc get events -o yaml
running test test-json-parsing.sh
++ set -o nounset
++ set -o pipefail
++ type get_running_pod
++ ARTIFACT_DIR=/tmp/origin-aggregated-logging/artifacts
++ '[' '!' -d /tmp/origin-aggregated-logging/artifacts ']'
+++ get_running_pod es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
+++ oc get pods -l component=es
++ es_pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ uuidgen
++ uuid_es=7651b91f-6111-41f2-ae3d-bf0a3344e918
++ echo Adding test message 7651b91f-6111-41f2-ae3d-bf0a3344e918 to Kibana . . .
Adding test message 7651b91f-6111-41f2-ae3d-bf0a3344e918 to Kibana . . .
++ add_test_message 7651b91f-6111-41f2-ae3d-bf0a3344e918
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-1fms4
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/7651b91f-6111-41f2-ae3d-bf0a3344e918
++ rc=0
++ timeout=600
++ echo Waiting 600 seconds for 7651b91f-6111-41f2-ae3d-bf0a3344e918 to show up in Elasticsearch . . .
Waiting 600 seconds for 7651b91f-6111-41f2-ae3d-bf0a3344e918 to show up in Elasticsearch . . .
++ espod=logging-es-data-master-f10vs1am-1-wm1tr
++ myproject=project.logging.
++ mymessage=7651b91f-6111-41f2-ae3d-bf0a3344e918
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 600
++ let ii=600
++ local interval=1
++ '[' 600 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging. _count message 7651b91f-6111-41f2-ae3d-bf0a3344e918
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 599 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging. _count message 7651b91f-6111-41f2-ae3d-bf0a3344e918
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 598 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging. _count message 7651b91f-6111-41f2-ae3d-bf0a3344e918
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918' --connect-timeout 1
+++ get_count_from_json
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
+++ shift
+++ shift
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 597 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging. _count message 7651b91f-6111-41f2-ae3d-bf0a3344e918
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
+++ shift
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 596 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging. _count message 7651b91f-6111-41f2-ae3d-bf0a3344e918
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 595 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging. _count message 7651b91f-6111-41f2-ae3d-bf0a3344e918
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 594 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging. _count message 7651b91f-6111-41f2-ae3d-bf0a3344e918
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 593 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging. _count message 7651b91f-6111-41f2-ae3d-bf0a3344e918
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 592 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging. _count message 7651b91f-6111-41f2-ae3d-bf0a3344e918
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 591 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging. _count message 7651b91f-6111-41f2-ae3d-bf0a3344e918
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 590 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging. _count message 7651b91f-6111-41f2-ae3d-bf0a3344e918
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 589 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging. _count message 7651b91f-6111-41f2-ae3d-bf0a3344e918
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 588 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging. _count message 7651b91f-6111-41f2-ae3d-bf0a3344e918
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 588 -le 0 ']'
++ return 0
good - ./logging.sh: found 1 record project logging for 7651b91f-6111-41f2-ae3d-bf0a3344e918
++ echo good - ./logging.sh: found 1 record project logging for 7651b91f-6111-41f2-ae3d-bf0a3344e918
Testing if record is in correct format . . .
++ echo Testing if record is in correct format . . .
++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging. _search message 7651b91f-6111-41f2-ae3d-bf0a3344e918
++ python test-json-parsing.py 7651b91f-6111-41f2-ae3d-bf0a3344e918
++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging.*/_search?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918' --connect-timeout 1
++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
++ local 'endpoint=/project.logging.*/_search?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
++ shift
++ shift
++ args=("${@:-}")
++ local args
++ local secret_dir=/etc/elasticsearch/secret/
++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_search?q=message:7651b91f-6111-41f2-ae3d-bf0a3344e918'
Success: record contains all of the expected fields/values
Success: ./logging.sh passed
++ echo Success: ./logging.sh passed
++ exit 0
running test test-mux.sh
++ set -o nounset
++ set -o pipefail
++ type get_running_pod
++ '[' false == false -o false == false ']'
Skipping -- This test requires both USE_MUX_CLIENT and MUX_ALLOW_EXTERNAL are true.
++ echo 'Skipping -- This test requires both USE_MUX_CLIENT and MUX_ALLOW_EXTERNAL are true.'
++ exit 0
SKIPPING upgrade test for now
running test test-viaq-data-model.sh
++ set -o nounset
++ set -o pipefail
++ type get_running_pod
++ [[ 1 -ne 1 ]]
++ [[ true = \f\a\l\s\e ]]
++ CLUSTER=true
++ ops=-ops
++ INDEX_PREFIX=
++ PROJ_PREFIX=project.
++ ARTIFACT_DIR=/tmp/origin-aggregated-logging/artifacts
++ '[' '!' -d /tmp/origin-aggregated-logging/artifacts ']'
++ get_test_user_token
++ local current_project
+++ oc project -q
++ current_project=logging
++ oc login --username=admin --password=admin
+++ oc whoami -t
++ test_token=QEkS2OJEBDw8HlvMBdWC8UUogjOCXviPzJkcMNzx_ng
+++ oc whoami
++ test_name=admin
++ test_ip=127.0.0.1
++ oc login --username=system:admin
++ oc project logging
++ TEST_DIVIDER=------------------------------------------
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-fnpvk
++ remove_test_volume
++ oc get template logging-fluentd-template -o json
++ python -c 'import json, sys; obj = json.loads(sys.stdin.read()); vm = obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"]; obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] = [xx for xx in vm if xx["name"] != "cdmtest"]; vs = obj["objects"][0]["spec"]["template"]["spec"]["volumes"]; obj["objects"][0]["spec"]["template"]["spec"]["volumes"] = [xx for xx in vs if xx["name"] != "cdmtest"]; print json.dumps(obj, indent=2)'
++ oc replace -f -
template "logging-fluentd-template" replaced
+++ mktemp
++ cfg=/tmp/tmp.T9bToOtNRI
++ cat
++ add_test_volume /tmp/tmp.T9bToOtNRI
++ oc get template logging-fluentd-template -o json
++ python -c 'import json, sys; obj = json.loads(sys.stdin.read()); obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"].append({"name": "cdmtest", "mountPath": "/etc/fluent/configs.d/openshift/filter-pre-cdm-test.conf", "readOnly": True}); obj["objects"][0]["spec"]["template"]["spec"]["volumes"].append({"name": "cdmtest", "hostPath": {"path": "/tmp/tmp.T9bToOtNRI"}}); print json.dumps(obj, indent=2)'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ trap cleanup INT TERM EXIT
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-fnpvk
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-fnpvk
++ '[' -z logging-fluentd-fnpvk -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-fnpvk
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-srr84
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-srr84 ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-srr84
++ keep_fields=method,statusCode,type,@timestamp,req,res
++ write_and_verify_logs test1
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1496970822
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-09 01:13:42.913502922+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 01:13:42.913502922+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
++ '[' -z logging-es-ops-data-master-gw0u0psf-1-j2gcm ']'
+++ uuidgen
++ local uuid_es=3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
+++ uuidgen
++ local uuid_es_ops=729d069e-d7e1-47db-b01a-c3245a5c8eda
++ local expected=1
++ local timeout=300
++ add_test_message 3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-1fms4
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
added es message 3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
++ echo added es message 3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
++ logger -i -p local6.info -t 729d069e-d7e1-47db-b01a-c3245a5c8eda 729d069e-d7e1-47db-b01a-c3245a5c8eda
added es-ops message 729d069e-d7e1-47db-b01a-c3245a5c8eda
++ echo added es-ops message 729d069e-d7e1-47db-b01a-c3245a5c8eda
++ local rc=0
++ espod=logging-es-data-master-f10vs1am-1-wm1tr
++ myproject=project.logging
++ mymessage=3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 292 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
++ espod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
++ myproject=.operations
++ mymessage=729d069e-d7e1-47db-b01a-c3245a5c8eda
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-gw0u0psf-1-j2gcm .operations _count systemd.u.SYSLOG_IDENTIFIER 729d069e-d7e1-47db-b01a-c3245a5c8eda
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-gw0u0psf-1-j2gcm '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:729d069e-d7e1-47db-b01a-c3245a5c8eda' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:729d069e-d7e1-47db-b01a-c3245a5c8eda'
+++ shift
+++ shift
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-gw0u0psf-1-j2gcm -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:729d069e-d7e1-47db-b01a-c3245a5c8eda'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-gw0u0psf-1-j2gcm .operations _count systemd.u.SYSLOG_IDENTIFIER 729d069e-d7e1-47db-b01a-c3245a5c8eda
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-gw0u0psf-1-j2gcm '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:729d069e-d7e1-47db-b01a-c3245a5c8eda' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:729d069e-d7e1-47db-b01a-c3245a5c8eda'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-gw0u0psf-1-j2gcm -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:729d069e-d7e1-47db-b01a-c3245a5c8eda'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-gw0u0psf-1-j2gcm .operations _count systemd.u.SYSLOG_IDENTIFIER 729d069e-d7e1-47db-b01a-c3245a5c8eda
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-gw0u0psf-1-j2gcm '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:729d069e-d7e1-47db-b01a-c3245a5c8eda' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:729d069e-d7e1-47db-b01a-c3245a5c8eda'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ oc exec logging-es-ops-data-master-gw0u0psf-1-j2gcm -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:729d069e-d7e1-47db-b01a-c3245a5c8eda'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-gw0u0psf-1-j2gcm .operations _count systemd.u.SYSLOG_IDENTIFIER 729d069e-d7e1-47db-b01a-c3245a5c8eda
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-gw0u0psf-1-j2gcm '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:729d069e-d7e1-47db-b01a-c3245a5c8eda' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:729d069e-d7e1-47db-b01a-c3245a5c8eda'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-gw0u0psf-1-j2gcm -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:729d069e-d7e1-47db-b01a-c3245a5c8eda'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 297 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 729d069e-d7e1-47db-b01a-c3245a5c8eda
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 729d069e-d7e1-47db-b01a-c3245a5c8eda
++ '[' -n get_logmessage ']'
++ get_logmessage 3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
++ logmessage=3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 729d069e-d7e1-47db-b01a-c3245a5c8eda
++ logmessage2=729d069e-d7e1-47db-b01a-c3245a5c8eda
+++ date +%s
++ local endtime=1496970839
+++ expr 1496970839 - 1496970822
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 17 seconds at 2017-06-09 01:13:59.899794631+00:00
++ echo END wait_for_fluentd_to_catch_up took 17 seconds at 2017-06-09 01:13:59.899794631+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-1fms4
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-1fms4 logging-es project.logging _search message 3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8
++ python test-viaq-data-model.py test1
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer QEkS2OJEBDw8HlvMBdWC8UUogjOCXviPzJkcMNzx_ng' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:3c3a7671-f3ab-4d5e-8bc9-1152df00cbb8'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-1fms4 logging-es-ops .operations _search message 729d069e-d7e1-47db-b01a-c3245a5c8eda
++ python test-viaq-data-model.py test1
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer QEkS2OJEBDw8HlvMBdWC8UUogjOCXviPzJkcMNzx_ng' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:729d069e-d7e1-47db-b01a-c3245a5c8eda'
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ add_cdm_env_var_val CDM_USE_UNDEFINED '"true"'
+++ mktemp
++ junk=/tmp/tmp.TZpkRlaatM
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.TZpkRlaatM'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.TZpkRlaatM
++ add_cdm_env_var_val CDM_EXTRA_KEEP_FIELDS method,statusCode,type,@timestamp,req,res
+++ mktemp
++ junk=/tmp/tmp.MCeoVQCU6p
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.MCeoVQCU6p'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.MCeoVQCU6p
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-srr84
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-srr84
++ '[' -z logging-fluentd-srr84 -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-srr84
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-prkdz
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-prkdz ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-prkdz
++ write_and_verify_logs test2
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1496970858
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-09 01:14:18.234708709+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 01:14:18.234708709+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
++ '[' -z logging-es-ops-data-master-gw0u0psf-1-j2gcm ']'
+++ uuidgen
++ local uuid_es=81d05305-5c45-4547-b887-bbf86ea27bdf
+++ uuidgen
++ local uuid_es_ops=0c3f530d-736e-4fff-8120-cf925d185ef3
++ local expected=1
++ local timeout=300
++ add_test_message 81d05305-5c45-4547-b887-bbf86ea27bdf
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-1fms4
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/81d05305-5c45-4547-b887-bbf86ea27bdf
++ echo added es message 81d05305-5c45-4547-b887-bbf86ea27bdf
added es message 81d05305-5c45-4547-b887-bbf86ea27bdf
++ logger -i -p local6.info -t 0c3f530d-736e-4fff-8120-cf925d185ef3 0c3f530d-736e-4fff-8120-cf925d185ef3
added es-ops message 0c3f530d-736e-4fff-8120-cf925d185ef3
++ echo added es-ops message 0c3f530d-736e-4fff-8120-cf925d185ef3
++ local rc=0
++ espod=logging-es-data-master-f10vs1am-1-wm1tr
++ myproject=project.logging
++ mymessage=81d05305-5c45-4547-b887-bbf86ea27bdf
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 81d05305-5c45-4547-b887-bbf86ea27bdf
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 81d05305-5c45-4547-b887-bbf86ea27bdf
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 81d05305-5c45-4547-b887-bbf86ea27bdf
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 81d05305-5c45-4547-b887-bbf86ea27bdf
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 81d05305-5c45-4547-b887-bbf86ea27bdf
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 81d05305-5c45-4547-b887-bbf86ea27bdf
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 81d05305-5c45-4547-b887-bbf86ea27bdf
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 81d05305-5c45-4547-b887-bbf86ea27bdf
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 81d05305-5c45-4547-b887-bbf86ea27bdf
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 292 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 81d05305-5c45-4547-b887-bbf86ea27bdf
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 81d05305-5c45-4547-b887-bbf86ea27bdf
++ espod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
++ myproject=.operations
++ mymessage=0c3f530d-736e-4fff-8120-cf925d185ef3
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-gw0u0psf-1-j2gcm .operations _count systemd.u.SYSLOG_IDENTIFIER 0c3f530d-736e-4fff-8120-cf925d185ef3
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-gw0u0psf-1-j2gcm '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:0c3f530d-736e-4fff-8120-cf925d185ef3' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:0c3f530d-736e-4fff-8120-cf925d185ef3'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-gw0u0psf-1-j2gcm -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:0c3f530d-736e-4fff-8120-cf925d185ef3'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-gw0u0psf-1-j2gcm .operations _count systemd.u.SYSLOG_IDENTIFIER 0c3f530d-736e-4fff-8120-cf925d185ef3
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-gw0u0psf-1-j2gcm '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:0c3f530d-736e-4fff-8120-cf925d185ef3' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:0c3f530d-736e-4fff-8120-cf925d185ef3'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-gw0u0psf-1-j2gcm -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:0c3f530d-736e-4fff-8120-cf925d185ef3'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-gw0u0psf-1-j2gcm .operations _count systemd.u.SYSLOG_IDENTIFIER 0c3f530d-736e-4fff-8120-cf925d185ef3
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-gw0u0psf-1-j2gcm '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:0c3f530d-736e-4fff-8120-cf925d185ef3' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:0c3f530d-736e-4fff-8120-cf925d185ef3'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-gw0u0psf-1-j2gcm -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:0c3f530d-736e-4fff-8120-cf925d185ef3'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 298 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 0c3f530d-736e-4fff-8120-cf925d185ef3
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 0c3f530d-736e-4fff-8120-cf925d185ef3
++ '[' -n get_logmessage ']'
++ get_logmessage 81d05305-5c45-4547-b887-bbf86ea27bdf
++ logmessage=81d05305-5c45-4547-b887-bbf86ea27bdf
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 0c3f530d-736e-4fff-8120-cf925d185ef3
++ logmessage2=0c3f530d-736e-4fff-8120-cf925d185ef3
+++ date +%s
++ local endtime=1496970873
+++ expr 1496970873 - 1496970858
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 15 seconds at 2017-06-09 01:14:33.767425070+00:00
++ echo END wait_for_fluentd_to_catch_up took 15 seconds at 2017-06-09 01:14:33.767425070+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-1fms4
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-1fms4 logging-es project.logging _search message 81d05305-5c45-4547-b887-bbf86ea27bdf
++ python test-viaq-data-model.py test2
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer QEkS2OJEBDw8HlvMBdWC8UUogjOCXviPzJkcMNzx_ng' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:81d05305-5c45-4547-b887-bbf86ea27bdf'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-1fms4 logging-es-ops .operations _search message 0c3f530d-736e-4fff-8120-cf925d185ef3
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer QEkS2OJEBDw8HlvMBdWC8UUogjOCXviPzJkcMNzx_ng' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:0c3f530d-736e-4fff-8120-cf925d185ef3'
++ python test-viaq-data-model.py test2
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ del_cdm_env_var CDM_EXTRA_KEEP_FIELDS
++ oc get template logging-fluentd-template -o yaml
++ sed '/- name: CDM_EXTRA_KEEP_FIELDS$/,/value:/d'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ add_cdm_env_var_val CDM_EXTRA_KEEP_FIELDS undefined4,undefined5,method,statusCode,type,@timestamp,req,res
+++ mktemp
++ junk=/tmp/tmp.j2IllIsnjF
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.j2IllIsnjF'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.j2IllIsnjF
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-prkdz
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-prkdz
++ '[' -z logging-fluentd-prkdz -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-prkdz
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
pod for component=fluentd not running yet
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-j2291
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-j2291 ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-j2291
++ write_and_verify_logs test3
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1496970900
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-09 01:15:00.788620079+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 01:15:00.788620079+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ get_running_pod es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
+++ oc get pods -l component=es-ops
++ local es_ops_pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
++ '[' -z logging-es-ops-data-master-gw0u0psf-1-j2gcm ']'
+++ uuidgen
++ local uuid_es=bee28c20-b0e6-430e-8363-06e780d78ac8
+++ uuidgen
++ local uuid_es_ops=89ac02f4-263b-40a3-bbd9-9ee2bea60f36
++ local expected=1
++ local timeout=300
++ add_test_message bee28c20-b0e6-430e-8363-06e780d78ac8
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-1fms4
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/bee28c20-b0e6-430e-8363-06e780d78ac8
added es message bee28c20-b0e6-430e-8363-06e780d78ac8
++ echo added es message bee28c20-b0e6-430e-8363-06e780d78ac8
++ logger -i -p local6.info -t 89ac02f4-263b-40a3-bbd9-9ee2bea60f36 89ac02f4-263b-40a3-bbd9-9ee2bea60f36
added es-ops message 89ac02f4-263b-40a3-bbd9-9ee2bea60f36
++ echo added es-ops message 89ac02f4-263b-40a3-bbd9-9ee2bea60f36
++ local rc=0
++ espod=logging-es-data-master-f10vs1am-1-wm1tr
++ myproject=project.logging
++ mymessage=bee28c20-b0e6-430e-8363-06e780d78ac8
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message bee28c20-b0e6-430e-8363-06e780d78ac8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message bee28c20-b0e6-430e-8363-06e780d78ac8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message bee28c20-b0e6-430e-8363-06e780d78ac8
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message bee28c20-b0e6-430e-8363-06e780d78ac8
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message bee28c20-b0e6-430e-8363-06e780d78ac8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message bee28c20-b0e6-430e-8363-06e780d78ac8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message bee28c20-b0e6-430e-8363-06e780d78ac8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message bee28c20-b0e6-430e-8363-06e780d78ac8
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message bee28c20-b0e6-430e-8363-06e780d78ac8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 291 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message bee28c20-b0e6-430e-8363-06e780d78ac8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 290 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message bee28c20-b0e6-430e-8363-06e780d78ac8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 289 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message bee28c20-b0e6-430e-8363-06e780d78ac8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 288 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message bee28c20-b0e6-430e-8363-06e780d78ac8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 287 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message bee28c20-b0e6-430e-8363-06e780d78ac8
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 287 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for bee28c20-b0e6-430e-8363-06e780d78ac8
good - wait_for_fluentd_to_catch_up: found 1 record project logging for bee28c20-b0e6-430e-8363-06e780d78ac8
++ espod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
++ myproject=.operations
++ mymessage=89ac02f4-263b-40a3-bbd9-9ee2bea60f36
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-gw0u0psf-1-j2gcm .operations _count systemd.u.SYSLOG_IDENTIFIER 89ac02f4-263b-40a3-bbd9-9ee2bea60f36
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-gw0u0psf-1-j2gcm '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:89ac02f4-263b-40a3-bbd9-9ee2bea60f36' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:89ac02f4-263b-40a3-bbd9-9ee2bea60f36'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-gw0u0psf-1-j2gcm -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:89ac02f4-263b-40a3-bbd9-9ee2bea60f36'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 89ac02f4-263b-40a3-bbd9-9ee2bea60f36
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 89ac02f4-263b-40a3-bbd9-9ee2bea60f36
++ '[' -n get_logmessage ']'
++ get_logmessage bee28c20-b0e6-430e-8363-06e780d78ac8
++ logmessage=bee28c20-b0e6-430e-8363-06e780d78ac8
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 89ac02f4-263b-40a3-bbd9-9ee2bea60f36
++ logmessage2=89ac02f4-263b-40a3-bbd9-9ee2bea60f36
+++ date +%s
++ local endtime=1496970920
+++ expr 1496970920 - 1496970900
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 20 seconds at 2017-06-09 01:15:20.643440849+00:00
++ echo END wait_for_fluentd_to_catch_up took 20 seconds at 2017-06-09 01:15:20.643440849+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-1fms4
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-1fms4 logging-es project.logging _search message bee28c20-b0e6-430e-8363-06e780d78ac8
++ python test-viaq-data-model.py test3
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer QEkS2OJEBDw8HlvMBdWC8UUogjOCXviPzJkcMNzx_ng' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:bee28c20-b0e6-430e-8363-06e780d78ac8'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-1fms4 logging-es-ops .operations _search message 89ac02f4-263b-40a3-bbd9-9ee2bea60f36
++ python test-viaq-data-model.py test3
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer QEkS2OJEBDw8HlvMBdWC8UUogjOCXviPzJkcMNzx_ng' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:89ac02f4-263b-40a3-bbd9-9ee2bea60f36'
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ add_cdm_env_var_val CDM_UNDEFINED_NAME myname
+++ mktemp
++ junk=/tmp/tmp.kD7u5RTrSF
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.kD7u5RTrSF'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.kD7u5RTrSF
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-j2291
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-j2291
++ '[' -z logging-fluentd-j2291 -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-j2291
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
pod for component=fluentd not running yet
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-xc9zr
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-xc9zr ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-xc9zr
++ write_and_verify_logs test4
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1496970937
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-09 01:15:37.666039906+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 01:15:37.666039906+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
++ '[' -z logging-es-ops-data-master-gw0u0psf-1-j2gcm ']'
+++ uuidgen
++ local uuid_es=6a8b2586-52da-42d4-b8ef-b2c5edc094da
+++ uuidgen
++ local uuid_es_ops=1c0f0d88-9867-4a3b-84aa-a597e04f1866
++ local expected=1
++ local timeout=300
++ add_test_message 6a8b2586-52da-42d4-b8ef-b2c5edc094da
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-1fms4
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/6a8b2586-52da-42d4-b8ef-b2c5edc094da
added es message 6a8b2586-52da-42d4-b8ef-b2c5edc094da
++ echo added es message 6a8b2586-52da-42d4-b8ef-b2c5edc094da
++ logger -i -p local6.info -t 1c0f0d88-9867-4a3b-84aa-a597e04f1866 1c0f0d88-9867-4a3b-84aa-a597e04f1866
added es-ops message 1c0f0d88-9867-4a3b-84aa-a597e04f1866
++ echo added es-ops message 1c0f0d88-9867-4a3b-84aa-a597e04f1866
++ local rc=0
++ espod=logging-es-data-master-f10vs1am-1-wm1tr
++ myproject=project.logging
++ mymessage=6a8b2586-52da-42d4-b8ef-b2c5edc094da
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 6a8b2586-52da-42d4-b8ef-b2c5edc094da
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 6a8b2586-52da-42d4-b8ef-b2c5edc094da
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 6a8b2586-52da-42d4-b8ef-b2c5edc094da
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 6a8b2586-52da-42d4-b8ef-b2c5edc094da
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 6a8b2586-52da-42d4-b8ef-b2c5edc094da
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 6a8b2586-52da-42d4-b8ef-b2c5edc094da
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 6a8b2586-52da-42d4-b8ef-b2c5edc094da
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 6a8b2586-52da-42d4-b8ef-b2c5edc094da
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message 6a8b2586-52da-42d4-b8ef-b2c5edc094da
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 292 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 6a8b2586-52da-42d4-b8ef-b2c5edc094da
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 6a8b2586-52da-42d4-b8ef-b2c5edc094da
++ espod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
++ myproject=.operations
++ mymessage=1c0f0d88-9867-4a3b-84aa-a597e04f1866
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-gw0u0psf-1-j2gcm .operations _count systemd.u.SYSLOG_IDENTIFIER 1c0f0d88-9867-4a3b-84aa-a597e04f1866
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-gw0u0psf-1-j2gcm '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:1c0f0d88-9867-4a3b-84aa-a597e04f1866' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:1c0f0d88-9867-4a3b-84aa-a597e04f1866'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-gw0u0psf-1-j2gcm -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:1c0f0d88-9867-4a3b-84aa-a597e04f1866'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-gw0u0psf-1-j2gcm .operations _count systemd.u.SYSLOG_IDENTIFIER 1c0f0d88-9867-4a3b-84aa-a597e04f1866
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-gw0u0psf-1-j2gcm '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:1c0f0d88-9867-4a3b-84aa-a597e04f1866' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:1c0f0d88-9867-4a3b-84aa-a597e04f1866'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-gw0u0psf-1-j2gcm -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:1c0f0d88-9867-4a3b-84aa-a597e04f1866'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-gw0u0psf-1-j2gcm .operations _count systemd.u.SYSLOG_IDENTIFIER 1c0f0d88-9867-4a3b-84aa-a597e04f1866
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-gw0u0psf-1-j2gcm '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:1c0f0d88-9867-4a3b-84aa-a597e04f1866' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:1c0f0d88-9867-4a3b-84aa-a597e04f1866'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-gw0u0psf-1-j2gcm -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:1c0f0d88-9867-4a3b-84aa-a597e04f1866'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-gw0u0psf-1-j2gcm .operations _count systemd.u.SYSLOG_IDENTIFIER 1c0f0d88-9867-4a3b-84aa-a597e04f1866
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-gw0u0psf-1-j2gcm '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:1c0f0d88-9867-4a3b-84aa-a597e04f1866' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:1c0f0d88-9867-4a3b-84aa-a597e04f1866'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-gw0u0psf-1-j2gcm -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:1c0f0d88-9867-4a3b-84aa-a597e04f1866'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 297 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 1c0f0d88-9867-4a3b-84aa-a597e04f1866
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 1c0f0d88-9867-4a3b-84aa-a597e04f1866
++ '[' -n get_logmessage ']'
++ get_logmessage 6a8b2586-52da-42d4-b8ef-b2c5edc094da
++ logmessage=6a8b2586-52da-42d4-b8ef-b2c5edc094da
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 1c0f0d88-9867-4a3b-84aa-a597e04f1866
++ logmessage2=1c0f0d88-9867-4a3b-84aa-a597e04f1866
+++ date +%s
++ local endtime=1496970954
+++ expr 1496970954 - 1496970937
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 17 seconds at 2017-06-09 01:15:54.683502737+00:00
++ echo END wait_for_fluentd_to_catch_up took 17 seconds at 2017-06-09 01:15:54.683502737+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-1fms4
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-1fms4 logging-es project.logging _search message 6a8b2586-52da-42d4-b8ef-b2c5edc094da
++ python test-viaq-data-model.py test4
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer QEkS2OJEBDw8HlvMBdWC8UUogjOCXviPzJkcMNzx_ng' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:6a8b2586-52da-42d4-b8ef-b2c5edc094da'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-1fms4 logging-es-ops .operations _search message 1c0f0d88-9867-4a3b-84aa-a597e04f1866
++ python test-viaq-data-model.py test4
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer QEkS2OJEBDw8HlvMBdWC8UUogjOCXviPzJkcMNzx_ng' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:1c0f0d88-9867-4a3b-84aa-a597e04f1866'
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ del_cdm_env_var CDM_EXTRA_KEEP_FIELDS
++ oc get template logging-fluentd-template -o yaml
++ sed '/- name: CDM_EXTRA_KEEP_FIELDS$/,/value:/d'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ add_cdm_env_var_val CDM_EXTRA_KEEP_FIELDS undefined4,undefined5,empty1,undefined3,method,statusCode,type,@timestamp,req,res
+++ mktemp
++ junk=/tmp/tmp.XPZxhLvTwQ
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.XPZxhLvTwQ'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.XPZxhLvTwQ
++ add_cdm_env_var_val CDM_KEEP_EMPTY_FIELDS undefined4,undefined5,empty1,undefined3
+++ mktemp
++ junk=/tmp/tmp.9KU1o1ZVNK
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.9KU1o1ZVNK'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.9KU1o1ZVNK
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-xc9zr
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-xc9zr
++ '[' -z logging-fluentd-xc9zr -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-xc9zr
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
pod for component=fluentd not running yet
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-vkg0x
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-vkg0x ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-vkg0x
++ write_and_verify_logs test5 allow_empty
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1496970980
+++ date -u --rfc-3339=ns
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 01:16:20.970588794+00:00
START wait_for_fluentd_to_catch_up at 2017-06-09 01:16:20.970588794+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
++ '[' -z logging-es-ops-data-master-gw0u0psf-1-j2gcm ']'
+++ uuidgen
++ local uuid_es=ccbb1f9e-4641-4e18-a43e-ca334867989e
+++ uuidgen
++ local uuid_es_ops=5c623caf-c522-42ca-b790-747589783898
++ local expected=1
++ local timeout=300
++ add_test_message ccbb1f9e-4641-4e18-a43e-ca334867989e
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-1fms4
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/ccbb1f9e-4641-4e18-a43e-ca334867989e
added es message ccbb1f9e-4641-4e18-a43e-ca334867989e
++ echo added es message ccbb1f9e-4641-4e18-a43e-ca334867989e
++ logger -i -p local6.info -t 5c623caf-c522-42ca-b790-747589783898 5c623caf-c522-42ca-b790-747589783898
added es-ops message 5c623caf-c522-42ca-b790-747589783898
++ echo added es-ops message 5c623caf-c522-42ca-b790-747589783898
++ local rc=0
++ espod=logging-es-data-master-f10vs1am-1-wm1tr
++ myproject=project.logging
++ mymessage=ccbb1f9e-4641-4e18-a43e-ca334867989e
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message ccbb1f9e-4641-4e18-a43e-ca334867989e
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message ccbb1f9e-4641-4e18-a43e-ca334867989e
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message ccbb1f9e-4641-4e18-a43e-ca334867989e
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
+++ shift
+++ shift
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message ccbb1f9e-4641-4e18-a43e-ca334867989e
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message ccbb1f9e-4641-4e18-a43e-ca334867989e
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message ccbb1f9e-4641-4e18-a43e-ca334867989e
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message ccbb1f9e-4641-4e18-a43e-ca334867989e
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message ccbb1f9e-4641-4e18-a43e-ca334867989e
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message ccbb1f9e-4641-4e18-a43e-ca334867989e
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e' --connect-timeout 1
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 291 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-f10vs1am-1-wm1tr project.logging _count message ccbb1f9e-4641-4e18-a43e-ca334867989e
+++ get_count_from_json
+++ curl_es logging-es-data-master-f10vs1am-1-wm1tr '/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-f10vs1am-1-wm1tr
+++ local 'endpoint=/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-f10vs1am-1-wm1tr -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 291 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project logging for ccbb1f9e-4641-4e18-a43e-ca334867989e
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for ccbb1f9e-4641-4e18-a43e-ca334867989e
++ espod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
++ myproject=.operations
++ mymessage=5c623caf-c522-42ca-b790-747589783898
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ get_count_from_json
+++ query_es_from_es logging-es-ops-data-master-gw0u0psf-1-j2gcm .operations _count systemd.u.SYSLOG_IDENTIFIER 5c623caf-c522-42ca-b790-747589783898
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-ops-data-master-gw0u0psf-1-j2gcm '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:5c623caf-c522-42ca-b790-747589783898' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-gw0u0psf-1-j2gcm
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:5c623caf-c522-42ca-b790-747589783898'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-gw0u0psf-1-j2gcm -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:5c623caf-c522-42ca-b790-747589783898'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 5c623caf-c522-42ca-b790-747589783898
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 5c623caf-c522-42ca-b790-747589783898
++ '[' -n get_logmessage ']'
++ get_logmessage ccbb1f9e-4641-4e18-a43e-ca334867989e
++ logmessage=ccbb1f9e-4641-4e18-a43e-ca334867989e
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 5c623caf-c522-42ca-b790-747589783898
++ logmessage2=5c623caf-c522-42ca-b790-747589783898
+++ date +%s
++ local endtime=1496970995
+++ expr 1496970995 - 1496970980
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 15 seconds at 2017-06-09 01:16:35.203540181+00:00
++ echo END wait_for_fluentd_to_catch_up took 15 seconds at 2017-06-09 01:16:35.203540181+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-1fms4
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-1fms4 logging-es project.logging _search message ccbb1f9e-4641-4e18-a43e-ca334867989e
++ python test-viaq-data-model.py test5 allow_empty
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer QEkS2OJEBDw8HlvMBdWC8UUogjOCXviPzJkcMNzx_ng' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:ccbb1f9e-4641-4e18-a43e-ca334867989e'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-1fms4 logging-es-ops .operations _search message 5c623caf-c522-42ca-b790-747589783898
++ python test-viaq-data-model.py test5 allow_empty
++ oc exec logging-kibana-1-1fms4 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer QEkS2OJEBDw8HlvMBdWC8UUogjOCXviPzJkcMNzx_ng' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:5c623caf-c522-42ca-b790-747589783898'
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ cleanup
++ remove_test_volume
++ oc get template logging-fluentd-template -o json
++ oc replace -f -
++ python -c 'import json, sys; obj = json.loads(sys.stdin.read()); vm = obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"]; obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] = [xx for xx in vm if xx["name"] != "cdmtest"]; vs = obj["objects"][0]["spec"]["template"]["spec"]["volumes"]; obj["objects"][0]["spec"]["template"]["spec"]["volumes"] = [xx for xx in vs if xx["name"] != "cdmtest"]; print json.dumps(obj, indent=2)'
template "logging-fluentd-template" replaced
++ remove_cdm_env
++ oc get template logging-fluentd-template -o yaml
++ sed '/- name: CDM_/,/value:/d'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.T9bToOtNRI
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-vkg0x
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-vkg0x
++ '[' -z logging-fluentd-vkg0x -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-vkg0x
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
SKIPPING reinstall test for now
++ curpod=logging-fluentd-76dxn
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-76dxn ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
/data/src/github.com/openshift/origin-aggregated-logging/hack/lib/log/system.sh: line 31:  4610 Terminated              sar -A -o "${binary_logfile}" 1 86400 > /dev/null 2> "${stderr_logfile}"  (wd: /data/src/github.com/openshift/origin-aggregated-logging)
[INFO] [CLEANUP] Beginning cleanup routines...
[INFO] [CLEANUP] Dumping cluster events to /tmp/origin-aggregated-logging/artifacts/events.txt
[INFO] [CLEANUP] Dumping etcd contents to /tmp/origin-aggregated-logging/artifacts/etcd
[WARNING] No compiled `etcdhelper` binary was found. Attempting to build one using:
[WARNING]   $ hack/build-go.sh tools/etcdhelper
++ Building go targets for linux/amd64: tools/etcdhelper
/data/src/github.com/openshift/origin-aggregated-logging/../origin/hack/build-go.sh took 140 seconds
2017-06-08 21:19:25.142930 I | warning: ignoring ServerName for user-provided CA for backwards compatibility is deprecated
[INFO] [CLEANUP] Dumping container logs to /tmp/origin-aggregated-logging/logs/containers
[INFO] [CLEANUP] Truncating log files over 200M
[INFO] [CLEANUP] Stopping docker containers
[INFO] [CLEANUP] Removing docker containers
Error: No such image, container or task: 789d2ca550e0
json: cannot unmarshal array into Go value of type types.ContainerJSON
Error response from daemon: You cannot remove a running container 32b118492aa44377eceea9254e623604c24740cdb35ec82749a6e3484b8ce546. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container d61c1a74ee65bb0460b3fb2c885e14242f91ab675dc7d034d2b01efdffc45c4c. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 9e2f0553d92eb644548b47ce35a58737461903b390c68263c3e8299af1d4e519. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container cff2eb237e35f940bcc6eedfd5db8d1a29f8e9995bd658161bbf2702a56d1a17. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 6223ee5634583222a7fdd768c3280b4ac1dd5d25985dd2083b43ecb0e6d09b17. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 0afe13eb3fad0130a90edbd7dfdbc27a1999cdb0ead0fc0c030cb30ccac63e55. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 9472cd2e28983e3f80d390fec8c35513bcef35921accc83c9abf5fa9b969784d. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container eac5be5e4147f22f7bd83e47b98e046d7a0daf56454487262443eb2aad44b93e. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 7770ada589fa2d5d51a3965b36ac6e17780c778d988af92235e413f3ed4e4f02. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 1285252ac71adf05b0382c1d7e02064a5589581b8197086876dd4efa08711d39. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 0d01c40a83c8bf6e397f4f5e2b7577359e9d1153ee0e4c29eb6b6d80bc058f3a. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container e971f63264584b523f40dbc5e355cc2cae4dd3d4df191f01177532247014494c. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 4b76a672c0da1468455512d30899fc3b65bbf6bdf074af28186c4f562aa0534d. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 61edfdb4b97dc4190474f2e14b2770169649411d66030728bfe7dc7346c44641. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 3a463ba96b76c00bb7141d1e65e04a6310202dcbdcb7ed773f1510cef75f3250. Stop the container before attempting removal or use -f
[INFO] [CLEANUP] Killing child processes
[INFO] [CLEANUP] Pruning etcd data directory
[INFO] /data/src/github.com/openshift/origin-aggregated-logging/logging.sh exited with code 0 after 00h 57m 30s

real	57m29.317s
user	5m22.860s
sys	0m59.121s
Finished GIT_URL=https://github.com/openshift/origin-aggregated-logging GIT_BRANCH=master O_A_L_DIR=/data/src/github.com/openshift/origin-aggregated-logging OS_ROOT=/data/src/github.com/openshift/origin ENABLE_OPS_CLUSTER=true USE_LOCAL_SOURCE=true TEST_PERF=false VERBOSE=1 OS_ANSIBLE_REPO=https://github.com/openshift/openshift-ansible OS_ANSIBLE_BRANCH=master ./logging.sh
***************************************************
==> openshiftdev: Downloading logs
==> openshiftdev: Downloading artifacts from '/var/log/yum.log' to '/var/lib/jenkins/jobs/test-origin-aggregated-logging/workspace/origin/artifacts/yum.log'
==> openshiftdev: Downloading artifacts from '/var/log/secure' to '/var/lib/jenkins/jobs/test-origin-aggregated-logging/workspace/origin/artifacts/secure'
==> openshiftdev: Downloading artifacts from '/var/log/audit/audit.log' to '/var/lib/jenkins/jobs/test-origin-aggregated-logging/workspace/origin/artifacts/audit.log'
==> openshiftdev: Downloading artifacts from '/tmp/origin-aggregated-logging/' to '/var/lib/jenkins/jobs/test-origin-aggregated-logging/workspace/origin/artifacts'
+ false
+ false
+ test_pull_requests --mark_test_success --repo origin-aggregated-logging --config /var/lib/jenkins/.test_pull_requests_logging.json
Rate limit remaining: 3313
  Marking SUCCESS for pull request #--repo in repo ''
/usr/share/ruby/net/http/response.rb:119:in `error!': 404 "Not Found" (Net::HTTPServerException)
	from /bin/test_pull_requests:543:in `block in get_comments'
	from /bin/test_pull_requests:535:in `each'
	from /bin/test_pull_requests:535:in `get_comments'
	from /bin/test_pull_requests:1074:in `get_comment_matching_regex'
	from /bin/test_pull_requests:1090:in `get_comment_with_prefix'
	from /bin/test_pull_requests:779:in `mark_test_success'
	from /bin/test_pull_requests:2361:in `<main>'
+ true
[description-setter] Could not determine description.
[PostBuildScript] - Execution post build scripts.
[workspace] $ /bin/sh -xe /tmp/hudson2164022223085475022.sh
+ INSTANCE_NAME=origin_logging-rhel7-1646
+ pushd origin
~/jobs/test-origin-aggregated-logging/workspace/origin ~/jobs/test-origin-aggregated-logging/workspace
+ rc=0
+ '[' -f .vagrant-openshift.json ']'
++ /usr/bin/vagrant ssh -c 'sudo ausearch -m avc'
+ ausearchresult='<no matches>'
+ rc=1
+ '[' '<no matches>' = '<no matches>' ']'
+ rc=0
+ /usr/bin/vagrant destroy -f
==> openshiftdev: Terminating the instance...
==> openshiftdev: Running cleanup tasks for 'shell' provisioner...
+ popd
~/jobs/test-origin-aggregated-logging/workspace
+ exit 0
Finished: SUCCESS