SuccessConsole Output

Skipping 615 KB.. Full Log
-7f73974f83cd'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:4ef1dfb3-fc3a-4e4e-91fb-7f73974f83cd'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 246 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 4ef1dfb3-fc3a-4e4e-91fb-7f73974f83cd
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 4ef1dfb3-fc3a-4e4e-91fb-7f73974f83cd
++ espod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
++ myproject=.operations
++ mymessage=84d2fe66-bcd9-4562-b0bc-58b37e179a59
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-kq3j3au5-1-8bzmq .operations _count systemd.u.SYSLOG_IDENTIFIER 84d2fe66-bcd9-4562-b0bc-58b37e179a59
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-kq3j3au5-1-8bzmq '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:84d2fe66-bcd9-4562-b0bc-58b37e179a59' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:84d2fe66-bcd9-4562-b0bc-58b37e179a59'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-kq3j3au5-1-8bzmq -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:84d2fe66-bcd9-4562-b0bc-58b37e179a59'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 84d2fe66-bcd9-4562-b0bc-58b37e179a59
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 84d2fe66-bcd9-4562-b0bc-58b37e179a59
++ '[' -n '' ']'
++ '[' -n '' ']'
+++ date +%s
++ local endtime=1497027893
+++ expr 1497027893 - 1497027816
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 77 seconds at 2017-06-09 17:04:53.580799825+00:00
++ echo END wait_for_fluentd_to_catch_up took 77 seconds at 2017-06-09 17:04:53.580799825+00:00
++ return 0
++ return 0
++ cleanup
++ cleanup_forward
++ oc label node --all logging-infra-fluentd-
node "172.18.12.108" labeled
++ wait_for_pod_ACTION stop logging-fluentd-26stf
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-26stf
++ '[' -z logging-fluentd-26stf -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-26stf
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc delete daemonset/logging-forward-fluentd
daemonset "logging-forward-fluentd" deleted
+++ grep '<match \*\*>'
+++ oc get configmap/logging-fluentd -o yaml
++ '[' -n '      <match **>' ']'
++ oc get configmap/logging-fluentd -o yaml
++ sed -e '/<match \*\*>/ d' -e '/@include configs\.d\/user\/secure-forward\.conf/ d' -e '/<\/match>/ d'
++ oc replace -f -
configmap "logging-fluentd" replaced
++ oc patch configmap/logging-fluentd --type=json --patch '[{ "op": "replace", "path": "/data/secure-forward.conf", "value": "\
# @type secure_forward\n\
# self_hostname forwarding-${HOSTNAME}\n\
# shared_key aggregated_logging_ci_testing\n\
#  secure no\n\
#  <server>\n\
#   host ${FLUENTD_FORWARD}\n\
#   port 24284\n\
#  </server>"}]'
configmap "logging-fluentd" patched
++ oc label node --all logging-infra-fluentd=true
node "172.18.12.108" labeled
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
pod for component=fluentd not running yet
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-fptx6
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-fptx6 ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-fptx6
++ oc get events -o yaml
++ write_and_verify_logs 1
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up '' ''
+++ date +%s
++ local starttime=1497027913
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-09 17:05:13.282831831+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 17:05:13.282831831+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-03l0u89m-1-p747h
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
++ '[' -z logging-es-ops-data-master-kq3j3au5-1-8bzmq ']'
+++ uuidgen
++ local uuid_es=14a14546-3450-4ca1-8ebc-4db5642c2524
+++ uuidgen
++ local uuid_es_ops=b31455dc-7230-4a2f-8181-be6fb791edd7
++ local expected=1
++ local timeout=300
++ add_test_message 14a14546-3450-4ca1-8ebc-4db5642c2524
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-20vz2
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/14a14546-3450-4ca1-8ebc-4db5642c2524
++ echo added es message 14a14546-3450-4ca1-8ebc-4db5642c2524
added es message 14a14546-3450-4ca1-8ebc-4db5642c2524
++ logger -i -p local6.info -t b31455dc-7230-4a2f-8181-be6fb791edd7 b31455dc-7230-4a2f-8181-be6fb791edd7
added es-ops message b31455dc-7230-4a2f-8181-be6fb791edd7
++ echo added es-ops message b31455dc-7230-4a2f-8181-be6fb791edd7
++ local rc=0
++ espod=logging-es-data-master-03l0u89m-1-p747h
++ myproject=project.logging
++ mymessage=14a14546-3450-4ca1-8ebc-4db5642c2524
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 14a14546-3450-4ca1-8ebc-4db5642c2524
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 14a14546-3450-4ca1-8ebc-4db5642c2524
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 14a14546-3450-4ca1-8ebc-4db5642c2524
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
+++ shift
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 14a14546-3450-4ca1-8ebc-4db5642c2524
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 14a14546-3450-4ca1-8ebc-4db5642c2524
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 14a14546-3450-4ca1-8ebc-4db5642c2524
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 14a14546-3450-4ca1-8ebc-4db5642c2524
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 14a14546-3450-4ca1-8ebc-4db5642c2524
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 14a14546-3450-4ca1-8ebc-4db5642c2524
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 291 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 14a14546-3450-4ca1-8ebc-4db5642c2524
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 290 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 14a14546-3450-4ca1-8ebc-4db5642c2524
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 289 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 14a14546-3450-4ca1-8ebc-4db5642c2524
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 288 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 14a14546-3450-4ca1-8ebc-4db5642c2524
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 287 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 14a14546-3450-4ca1-8ebc-4db5642c2524
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:14a14546-3450-4ca1-8ebc-4db5642c2524'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 287 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 14a14546-3450-4ca1-8ebc-4db5642c2524
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 14a14546-3450-4ca1-8ebc-4db5642c2524
++ espod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
++ myproject=.operations
++ mymessage=b31455dc-7230-4a2f-8181-be6fb791edd7
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-kq3j3au5-1-8bzmq .operations _count systemd.u.SYSLOG_IDENTIFIER b31455dc-7230-4a2f-8181-be6fb791edd7
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-kq3j3au5-1-8bzmq '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:b31455dc-7230-4a2f-8181-be6fb791edd7' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:b31455dc-7230-4a2f-8181-be6fb791edd7'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-kq3j3au5-1-8bzmq -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:b31455dc-7230-4a2f-8181-be6fb791edd7'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for b31455dc-7230-4a2f-8181-be6fb791edd7
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for b31455dc-7230-4a2f-8181-be6fb791edd7
++ '[' -n '' ']'
++ '[' -n '' ']'
+++ date +%s
++ local endtime=1497027933
+++ expr 1497027933 - 1497027913
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 20 seconds at 2017-06-09 17:05:33.008995954+00:00
++ echo END wait_for_fluentd_to_catch_up took 20 seconds at 2017-06-09 17:05:33.008995954+00:00
++ return 0
++ return 0
++ cleanup
++ cleanup_forward
++ oc label node --all logging-infra-fluentd-
node "172.18.12.108" labeled
++ wait_for_pod_ACTION stop logging-fluentd-fptx6
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-fptx6
++ '[' -z logging-fluentd-fptx6 -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-fptx6
++ '[' -n 1 ']'
pod logging-fluentd-fptx6 still running
++ echo pod logging-fluentd-fptx6 still running
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' stop = start ']'
++ '[' 110 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-fptx6
++ '[' stop = start ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
++ oc delete daemonset/logging-forward-fluentd
Error from server (NotFound): daemonsets.extensions "logging-forward-fluentd" not found
++ :
+++ oc get configmap/logging-fluentd -o yaml
+++ grep '<match \*\*>'
++ '[' -n '' ']'
++ oc patch configmap/logging-fluentd --type=json --patch '[{ "op": "replace", "path": "/data/secure-forward.conf", "value": "\
# @type secure_forward\n\
# self_hostname forwarding-${HOSTNAME}\n\
# shared_key aggregated_logging_ci_testing\n\
#  secure no\n\
#  <server>\n\
#   host ${FLUENTD_FORWARD}\n\
#   port 24284\n\
#  </server>"}]'
configmap "logging-fluentd" not patched
++ oc label node --all logging-infra-fluentd=true
node "172.18.12.108" labeled
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-wlspv
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-wlspv ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
+++ oc get pods -l component=fluentd
++ fpod=logging-fluentd-wlspv
++ oc get events -o yaml
running test test-json-parsing.sh
++ set -o nounset
++ set -o pipefail
++ type get_running_pod
++ ARTIFACT_DIR=/tmp/origin-aggregated-logging/artifacts
++ '[' '!' -d /tmp/origin-aggregated-logging/artifacts ']'
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ es_pod=logging-es-data-master-03l0u89m-1-p747h
+++ uuidgen
++ uuid_es=f0bdebdd-82e5-4aa7-a864-6190b9b2e389
++ echo Adding test message f0bdebdd-82e5-4aa7-a864-6190b9b2e389 to Kibana . . .
Adding test message f0bdebdd-82e5-4aa7-a864-6190b9b2e389 to Kibana . . .
++ add_test_message f0bdebdd-82e5-4aa7-a864-6190b9b2e389
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-20vz2
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/f0bdebdd-82e5-4aa7-a864-6190b9b2e389
++ rc=0
++ timeout=600
++ echo Waiting 600 seconds for f0bdebdd-82e5-4aa7-a864-6190b9b2e389 to show up in Elasticsearch . . .
Waiting 600 seconds for f0bdebdd-82e5-4aa7-a864-6190b9b2e389 to show up in Elasticsearch . . .
++ espod=logging-es-data-master-03l0u89m-1-p747h
++ myproject=project.logging.
++ mymessage=f0bdebdd-82e5-4aa7-a864-6190b9b2e389
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 600
++ let ii=600
++ local interval=1
++ '[' 600 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging. _count message f0bdebdd-82e5-4aa7-a864-6190b9b2e389
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 599 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging. _count message f0bdebdd-82e5-4aa7-a864-6190b9b2e389
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 598 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging. _count message f0bdebdd-82e5-4aa7-a864-6190b9b2e389
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 597 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging. _count message f0bdebdd-82e5-4aa7-a864-6190b9b2e389
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 596 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging. _count message f0bdebdd-82e5-4aa7-a864-6190b9b2e389
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 595 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging. _count message f0bdebdd-82e5-4aa7-a864-6190b9b2e389
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 594 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging. _count message f0bdebdd-82e5-4aa7-a864-6190b9b2e389
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 593 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging. _count message f0bdebdd-82e5-4aa7-a864-6190b9b2e389
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 592 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging. _count message f0bdebdd-82e5-4aa7-a864-6190b9b2e389
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 591 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging. _count message f0bdebdd-82e5-4aa7-a864-6190b9b2e389
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 590 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging. _count message f0bdebdd-82e5-4aa7-a864-6190b9b2e389
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 589 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging. _count message f0bdebdd-82e5-4aa7-a864-6190b9b2e389
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 589 -le 0 ']'
++ return 0
good - ./logging.sh: found 1 record project logging for f0bdebdd-82e5-4aa7-a864-6190b9b2e389
++ echo good - ./logging.sh: found 1 record project logging for f0bdebdd-82e5-4aa7-a864-6190b9b2e389
Testing if record is in correct format . . .
++ echo Testing if record is in correct format . . .
++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging. _search message f0bdebdd-82e5-4aa7-a864-6190b9b2e389
++ python test-json-parsing.py f0bdebdd-82e5-4aa7-a864-6190b9b2e389
++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging.*/_search?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389' --connect-timeout 1
++ local pod=logging-es-data-master-03l0u89m-1-p747h
++ local 'endpoint=/project.logging.*/_search?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
++ shift
++ shift
++ args=("${@:-}")
++ local args
++ local secret_dir=/etc/elasticsearch/secret/
++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_search?q=message:f0bdebdd-82e5-4aa7-a864-6190b9b2e389'
Success: record contains all of the expected fields/values
Success: ./logging.sh passed
++ echo Success: ./logging.sh passed
++ exit 0
running test test-mux.sh
++ set -o nounset
++ set -o pipefail
++ type get_running_pod
++ '[' false == false -o false == false ']'
Skipping -- This test requires both USE_MUX_CLIENT and MUX_ALLOW_EXTERNAL are true.
++ echo 'Skipping -- This test requires both USE_MUX_CLIENT and MUX_ALLOW_EXTERNAL are true.'
++ exit 0
SKIPPING upgrade test for now
running test test-viaq-data-model.sh
++ set -o nounset
++ set -o pipefail
++ type get_running_pod
++ [[ 1 -ne 1 ]]
++ [[ true = \f\a\l\s\e ]]
++ CLUSTER=true
++ ops=-ops
++ INDEX_PREFIX=
++ PROJ_PREFIX=project.
++ ARTIFACT_DIR=/tmp/origin-aggregated-logging/artifacts
++ '[' '!' -d /tmp/origin-aggregated-logging/artifacts ']'
++ get_test_user_token
++ local current_project
+++ oc project -q
++ current_project=logging
++ oc login --username=admin --password=admin
+++ oc whoami -t
++ test_token=gq0SLvitkDNWYLytx7I6FpfYNPl8zvvHEE_r36Gr3lA
+++ oc whoami
++ test_name=admin
++ test_ip=127.0.0.1
++ oc login --username=system:admin
++ oc project logging
++ TEST_DIVIDER=------------------------------------------
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-wlspv
++ remove_test_volume
++ oc get template logging-fluentd-template -o json
++ python -c 'import json, sys; obj = json.loads(sys.stdin.read()); vm = obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"]; obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] = [xx for xx in vm if xx["name"] != "cdmtest"]; vs = obj["objects"][0]["spec"]["template"]["spec"]["volumes"]; obj["objects"][0]["spec"]["template"]["spec"]["volumes"] = [xx for xx in vs if xx["name"] != "cdmtest"]; print json.dumps(obj, indent=2)'
++ oc replace -f -
template "logging-fluentd-template" replaced
+++ mktemp
++ cfg=/tmp/tmp.RDolQxko64
++ cat
++ add_test_volume /tmp/tmp.RDolQxko64
++ oc get template logging-fluentd-template -o json
++ python -c 'import json, sys; obj = json.loads(sys.stdin.read()); obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"].append({"name": "cdmtest", "mountPath": "/etc/fluent/configs.d/openshift/filter-pre-cdm-test.conf", "readOnly": True}); obj["objects"][0]["spec"]["template"]["spec"]["volumes"].append({"name": "cdmtest", "hostPath": {"path": "/tmp/tmp.RDolQxko64"}}); print json.dumps(obj, indent=2)'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ trap cleanup INT TERM EXIT
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-wlspv
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-wlspv
++ '[' -z logging-fluentd-wlspv -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-wlspv
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-jp70j
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-jp70j ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-jp70j
++ keep_fields=method,statusCode,type,@timestamp,req,res
++ write_and_verify_logs test1
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1497027992
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-09 17:06:32.398679961+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 17:06:32.398679961+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-03l0u89m-1-p747h
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
++ '[' -z logging-es-ops-data-master-kq3j3au5-1-8bzmq ']'
+++ uuidgen
++ local uuid_es=65ebea43-3d53-4af4-b80d-5d23fe1a4e27
+++ uuidgen
++ local uuid_es_ops=7168732c-49e6-4d5b-bb43-4d2f46ace509
++ local expected=1
++ local timeout=300
++ add_test_message 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-20vz2
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/65ebea43-3d53-4af4-b80d-5d23fe1a4e27
added es message 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
++ echo added es message 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
++ logger -i -p local6.info -t 7168732c-49e6-4d5b-bb43-4d2f46ace509 7168732c-49e6-4d5b-bb43-4d2f46ace509
added es-ops message 7168732c-49e6-4d5b-bb43-4d2f46ace509
++ echo added es-ops message 7168732c-49e6-4d5b-bb43-4d2f46ace509
++ local rc=0
++ espod=logging-es-data-master-03l0u89m-1-p747h
++ myproject=project.logging
++ mymessage=65ebea43-3d53-4af4-b80d-5d23fe1a4e27
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 291 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 290 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 290 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
++ espod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
++ myproject=.operations
++ mymessage=7168732c-49e6-4d5b-bb43-4d2f46ace509
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-kq3j3au5-1-8bzmq .operations _count systemd.u.SYSLOG_IDENTIFIER 7168732c-49e6-4d5b-bb43-4d2f46ace509
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-kq3j3au5-1-8bzmq '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:7168732c-49e6-4d5b-bb43-4d2f46ace509' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:7168732c-49e6-4d5b-bb43-4d2f46ace509'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-kq3j3au5-1-8bzmq -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:7168732c-49e6-4d5b-bb43-4d2f46ace509'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 7168732c-49e6-4d5b-bb43-4d2f46ace509
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 7168732c-49e6-4d5b-bb43-4d2f46ace509
++ '[' -n get_logmessage ']'
++ get_logmessage 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
++ logmessage=65ebea43-3d53-4af4-b80d-5d23fe1a4e27
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 7168732c-49e6-4d5b-bb43-4d2f46ace509
++ logmessage2=7168732c-49e6-4d5b-bb43-4d2f46ace509
+++ date +%s
++ local endtime=1497028007
+++ expr 1497028007 - 1497027992
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 15 seconds at 2017-06-09 17:06:47.988584526+00:00
++ echo END wait_for_fluentd_to_catch_up took 15 seconds at 2017-06-09 17:06:47.988584526+00:00
++ return 0
+++ get_running_pod kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
+++ oc get pods -l component=kibana
++ kpod=logging-kibana-1-20vz2
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-20vz2 logging-es project.logging _search message 65ebea43-3d53-4af4-b80d-5d23fe1a4e27
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer gq0SLvitkDNWYLytx7I6FpfYNPl8zvvHEE_r36Gr3lA' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:65ebea43-3d53-4af4-b80d-5d23fe1a4e27'
++ python test-viaq-data-model.py test1
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-20vz2 logging-es-ops .operations _search message 7168732c-49e6-4d5b-bb43-4d2f46ace509
++ python test-viaq-data-model.py test1
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer gq0SLvitkDNWYLytx7I6FpfYNPl8zvvHEE_r36Gr3lA' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:7168732c-49e6-4d5b-bb43-4d2f46ace509'
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ add_cdm_env_var_val CDM_USE_UNDEFINED '"true"'
+++ mktemp
++ junk=/tmp/tmp.lfhSKXtVxx
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.lfhSKXtVxx'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.lfhSKXtVxx
++ add_cdm_env_var_val CDM_EXTRA_KEEP_FIELDS method,statusCode,type,@timestamp,req,res
+++ mktemp
++ junk=/tmp/tmp.Zd72FTurmG
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.Zd72FTurmG'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.Zd72FTurmG
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-jp70j
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-jp70j
++ '[' -z logging-fluentd-jp70j -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-jp70j
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
pod for component=fluentd not running yet
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-2hzht
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-2hzht ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-2hzht
++ write_and_verify_logs test2
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1497028032
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-09 17:07:12.747955447+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 17:07:12.747955447+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-03l0u89m-1-p747h
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
++ '[' -z logging-es-ops-data-master-kq3j3au5-1-8bzmq ']'
+++ uuidgen
++ local uuid_es=b389a17a-eeb3-4979-a15d-723888f2bb65
+++ uuidgen
++ local uuid_es_ops=c1eb0d52-5f45-4312-b34f-1e9983ce8530
++ local expected=1
++ local timeout=300
++ add_test_message b389a17a-eeb3-4979-a15d-723888f2bb65
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-20vz2
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/b389a17a-eeb3-4979-a15d-723888f2bb65
added es message b389a17a-eeb3-4979-a15d-723888f2bb65
++ echo added es message b389a17a-eeb3-4979-a15d-723888f2bb65
++ logger -i -p local6.info -t c1eb0d52-5f45-4312-b34f-1e9983ce8530 c1eb0d52-5f45-4312-b34f-1e9983ce8530
added es-ops message c1eb0d52-5f45-4312-b34f-1e9983ce8530
++ echo added es-ops message c1eb0d52-5f45-4312-b34f-1e9983ce8530
++ local rc=0
++ espod=logging-es-data-master-03l0u89m-1-p747h
++ myproject=project.logging
++ mymessage=b389a17a-eeb3-4979-a15d-723888f2bb65
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message b389a17a-eeb3-4979-a15d-723888f2bb65
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message b389a17a-eeb3-4979-a15d-723888f2bb65
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message b389a17a-eeb3-4979-a15d-723888f2bb65
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message b389a17a-eeb3-4979-a15d-723888f2bb65
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message b389a17a-eeb3-4979-a15d-723888f2bb65
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message b389a17a-eeb3-4979-a15d-723888f2bb65
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
+++ shift
+++ shift
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message b389a17a-eeb3-4979-a15d-723888f2bb65
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message b389a17a-eeb3-4979-a15d-723888f2bb65
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message b389a17a-eeb3-4979-a15d-723888f2bb65
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 291 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message b389a17a-eeb3-4979-a15d-723888f2bb65
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 290 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message b389a17a-eeb3-4979-a15d-723888f2bb65
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 289 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message b389a17a-eeb3-4979-a15d-723888f2bb65
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 288 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message b389a17a-eeb3-4979-a15d-723888f2bb65
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 287 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message b389a17a-eeb3-4979-a15d-723888f2bb65
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 287 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for b389a17a-eeb3-4979-a15d-723888f2bb65
good - wait_for_fluentd_to_catch_up: found 1 record project logging for b389a17a-eeb3-4979-a15d-723888f2bb65
++ espod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
++ myproject=.operations
++ mymessage=c1eb0d52-5f45-4312-b34f-1e9983ce8530
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-kq3j3au5-1-8bzmq .operations _count systemd.u.SYSLOG_IDENTIFIER c1eb0d52-5f45-4312-b34f-1e9983ce8530
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-kq3j3au5-1-8bzmq '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:c1eb0d52-5f45-4312-b34f-1e9983ce8530' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:c1eb0d52-5f45-4312-b34f-1e9983ce8530'
+++ shift
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-kq3j3au5-1-8bzmq -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:c1eb0d52-5f45-4312-b34f-1e9983ce8530'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for c1eb0d52-5f45-4312-b34f-1e9983ce8530
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for c1eb0d52-5f45-4312-b34f-1e9983ce8530
++ '[' -n get_logmessage ']'
++ get_logmessage b389a17a-eeb3-4979-a15d-723888f2bb65
++ logmessage=b389a17a-eeb3-4979-a15d-723888f2bb65
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 c1eb0d52-5f45-4312-b34f-1e9983ce8530
++ logmessage2=c1eb0d52-5f45-4312-b34f-1e9983ce8530
+++ date +%s
++ local endtime=1497028053
+++ expr 1497028053 - 1497028032
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 21 seconds at 2017-06-09 17:07:33.032434180+00:00
++ echo END wait_for_fluentd_to_catch_up took 21 seconds at 2017-06-09 17:07:33.032434180+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-20vz2
++ '[' 0 = 0 ']'
++ python test-viaq-data-model.py test2
++ curl_es_from_kibana logging-kibana-1-20vz2 logging-es project.logging _search message b389a17a-eeb3-4979-a15d-723888f2bb65
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer gq0SLvitkDNWYLytx7I6FpfYNPl8zvvHEE_r36Gr3lA' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:b389a17a-eeb3-4979-a15d-723888f2bb65'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-20vz2 logging-es-ops .operations _search message c1eb0d52-5f45-4312-b34f-1e9983ce8530
++ python test-viaq-data-model.py test2
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer gq0SLvitkDNWYLytx7I6FpfYNPl8zvvHEE_r36Gr3lA' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:c1eb0d52-5f45-4312-b34f-1e9983ce8530'
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ del_cdm_env_var CDM_EXTRA_KEEP_FIELDS
++ oc get template logging-fluentd-template -o yaml
++ sed '/- name: CDM_EXTRA_KEEP_FIELDS$/,/value:/d'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ add_cdm_env_var_val CDM_EXTRA_KEEP_FIELDS undefined4,undefined5,method,statusCode,type,@timestamp,req,res
+++ mktemp
++ junk=/tmp/tmp.iybK4F5SSr
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.iybK4F5SSr'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.iybK4F5SSr
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-2hzht
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-2hzht
++ '[' -z logging-fluentd-2hzht -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-2hzht
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
pod for component=fluentd not running yet
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-8kn33
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-8kn33 ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-8kn33
++ write_and_verify_logs test3
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1497028083
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-09 17:08:03.093894993+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 17:08:03.093894993+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-03l0u89m-1-p747h
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
++ '[' -z logging-es-ops-data-master-kq3j3au5-1-8bzmq ']'
+++ uuidgen
++ local uuid_es=3969390e-78ab-451c-b193-049a8b1dc50f
+++ uuidgen
++ local uuid_es_ops=4852e26f-a8d7-410f-81fc-696a168be4b0
++ local expected=1
++ local timeout=300
++ add_test_message 3969390e-78ab-451c-b193-049a8b1dc50f
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-20vz2
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/3969390e-78ab-451c-b193-049a8b1dc50f
added es message 3969390e-78ab-451c-b193-049a8b1dc50f
++ echo added es message 3969390e-78ab-451c-b193-049a8b1dc50f
++ logger -i -p local6.info -t 4852e26f-a8d7-410f-81fc-696a168be4b0 4852e26f-a8d7-410f-81fc-696a168be4b0
added es-ops message 4852e26f-a8d7-410f-81fc-696a168be4b0
++ echo added es-ops message 4852e26f-a8d7-410f-81fc-696a168be4b0
++ local rc=0
++ espod=logging-es-data-master-03l0u89m-1-p747h
++ myproject=project.logging
++ mymessage=3969390e-78ab-451c-b193-049a8b1dc50f
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 3969390e-78ab-451c-b193-049a8b1dc50f
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 3969390e-78ab-451c-b193-049a8b1dc50f
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
+++ shift
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 3969390e-78ab-451c-b193-049a8b1dc50f
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 3969390e-78ab-451c-b193-049a8b1dc50f
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 3969390e-78ab-451c-b193-049a8b1dc50f
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ get_count_from_json
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 3969390e-78ab-451c-b193-049a8b1dc50f
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 3969390e-78ab-451c-b193-049a8b1dc50f
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 3969390e-78ab-451c-b193-049a8b1dc50f
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 3969390e-78ab-451c-b193-049a8b1dc50f
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 291 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message 3969390e-78ab-451c-b193-049a8b1dc50f
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 291 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 3969390e-78ab-451c-b193-049a8b1dc50f
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 3969390e-78ab-451c-b193-049a8b1dc50f
++ espod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
++ myproject=.operations
++ mymessage=4852e26f-a8d7-410f-81fc-696a168be4b0
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-kq3j3au5-1-8bzmq .operations _count systemd.u.SYSLOG_IDENTIFIER 4852e26f-a8d7-410f-81fc-696a168be4b0
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-kq3j3au5-1-8bzmq '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:4852e26f-a8d7-410f-81fc-696a168be4b0' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:4852e26f-a8d7-410f-81fc-696a168be4b0'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-kq3j3au5-1-8bzmq -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:4852e26f-a8d7-410f-81fc-696a168be4b0'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-kq3j3au5-1-8bzmq .operations _count systemd.u.SYSLOG_IDENTIFIER 4852e26f-a8d7-410f-81fc-696a168be4b0
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-kq3j3au5-1-8bzmq '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:4852e26f-a8d7-410f-81fc-696a168be4b0' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:4852e26f-a8d7-410f-81fc-696a168be4b0'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-kq3j3au5-1-8bzmq -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:4852e26f-a8d7-410f-81fc-696a168be4b0'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-kq3j3au5-1-8bzmq .operations _count systemd.u.SYSLOG_IDENTIFIER 4852e26f-a8d7-410f-81fc-696a168be4b0
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-ops-data-master-kq3j3au5-1-8bzmq '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:4852e26f-a8d7-410f-81fc-696a168be4b0' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:4852e26f-a8d7-410f-81fc-696a168be4b0'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-kq3j3au5-1-8bzmq -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:4852e26f-a8d7-410f-81fc-696a168be4b0'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-kq3j3au5-1-8bzmq .operations _count systemd.u.SYSLOG_IDENTIFIER 4852e26f-a8d7-410f-81fc-696a168be4b0
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-kq3j3au5-1-8bzmq '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:4852e26f-a8d7-410f-81fc-696a168be4b0' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:4852e26f-a8d7-410f-81fc-696a168be4b0'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-kq3j3au5-1-8bzmq -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:4852e26f-a8d7-410f-81fc-696a168be4b0'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 297 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 4852e26f-a8d7-410f-81fc-696a168be4b0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 4852e26f-a8d7-410f-81fc-696a168be4b0
++ '[' -n get_logmessage ']'
++ get_logmessage 3969390e-78ab-451c-b193-049a8b1dc50f
++ logmessage=3969390e-78ab-451c-b193-049a8b1dc50f
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 4852e26f-a8d7-410f-81fc-696a168be4b0
++ logmessage2=4852e26f-a8d7-410f-81fc-696a168be4b0
+++ date +%s
++ local endtime=1497028101
+++ expr 1497028101 - 1497028083
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 18 seconds at 2017-06-09 17:08:21.778546655+00:00
++ echo END wait_for_fluentd_to_catch_up took 18 seconds at 2017-06-09 17:08:21.778546655+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-20vz2
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-20vz2 logging-es project.logging _search message 3969390e-78ab-451c-b193-049a8b1dc50f
++ python test-viaq-data-model.py test3
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer gq0SLvitkDNWYLytx7I6FpfYNPl8zvvHEE_r36Gr3lA' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:3969390e-78ab-451c-b193-049a8b1dc50f'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-20vz2 logging-es-ops .operations _search message 4852e26f-a8d7-410f-81fc-696a168be4b0
++ python test-viaq-data-model.py test3
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer gq0SLvitkDNWYLytx7I6FpfYNPl8zvvHEE_r36Gr3lA' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:4852e26f-a8d7-410f-81fc-696a168be4b0'
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ add_cdm_env_var_val CDM_UNDEFINED_NAME myname
+++ mktemp
++ junk=/tmp/tmp.45FCBqG641
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.45FCBqG641'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.45FCBqG641
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-8kn33
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-8kn33
++ '[' -z logging-fluentd-8kn33 -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-8kn33
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
pod for component=fluentd not running yet
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-j07sl
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-j07sl ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-j07sl
++ write_and_verify_logs test4
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1497028122
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-09 17:08:42.870313947+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 17:08:42.870313947+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-03l0u89m-1-p747h
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
++ '[' -z logging-es-ops-data-master-kq3j3au5-1-8bzmq ']'
+++ uuidgen
++ local uuid_es=bc03c63b-5593-4717-b304-cc81085a12d5
+++ uuidgen
++ local uuid_es_ops=337a6519-d385-4768-a421-599d6dc5c462
++ local expected=1
++ local timeout=300
++ add_test_message bc03c63b-5593-4717-b304-cc81085a12d5
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-20vz2
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/bc03c63b-5593-4717-b304-cc81085a12d5
added es message bc03c63b-5593-4717-b304-cc81085a12d5
++ echo added es message bc03c63b-5593-4717-b304-cc81085a12d5
++ logger -i -p local6.info -t 337a6519-d385-4768-a421-599d6dc5c462 337a6519-d385-4768-a421-599d6dc5c462
added es-ops message 337a6519-d385-4768-a421-599d6dc5c462
++ echo added es-ops message 337a6519-d385-4768-a421-599d6dc5c462
++ local rc=0
++ espod=logging-es-data-master-03l0u89m-1-p747h
++ myproject=project.logging
++ mymessage=bc03c63b-5593-4717-b304-cc81085a12d5
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message bc03c63b-5593-4717-b304-cc81085a12d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message bc03c63b-5593-4717-b304-cc81085a12d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message bc03c63b-5593-4717-b304-cc81085a12d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message bc03c63b-5593-4717-b304-cc81085a12d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message bc03c63b-5593-4717-b304-cc81085a12d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message bc03c63b-5593-4717-b304-cc81085a12d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message bc03c63b-5593-4717-b304-cc81085a12d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message bc03c63b-5593-4717-b304-cc81085a12d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message bc03c63b-5593-4717-b304-cc81085a12d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
+++ shift
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 291 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message bc03c63b-5593-4717-b304-cc81085a12d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 290 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message bc03c63b-5593-4717-b304-cc81085a12d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 290 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for bc03c63b-5593-4717-b304-cc81085a12d5
good - wait_for_fluentd_to_catch_up: found 1 record project logging for bc03c63b-5593-4717-b304-cc81085a12d5
++ espod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
++ myproject=.operations
++ mymessage=337a6519-d385-4768-a421-599d6dc5c462
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-kq3j3au5-1-8bzmq .operations _count systemd.u.SYSLOG_IDENTIFIER 337a6519-d385-4768-a421-599d6dc5c462
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-kq3j3au5-1-8bzmq '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:337a6519-d385-4768-a421-599d6dc5c462' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:337a6519-d385-4768-a421-599d6dc5c462'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-kq3j3au5-1-8bzmq -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:337a6519-d385-4768-a421-599d6dc5c462'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 337a6519-d385-4768-a421-599d6dc5c462
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 337a6519-d385-4768-a421-599d6dc5c462
++ '[' -n get_logmessage ']'
++ get_logmessage bc03c63b-5593-4717-b304-cc81085a12d5
++ logmessage=bc03c63b-5593-4717-b304-cc81085a12d5
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 337a6519-d385-4768-a421-599d6dc5c462
++ logmessage2=337a6519-d385-4768-a421-599d6dc5c462
+++ date +%s
++ local endtime=1497028138
+++ expr 1497028138 - 1497028122
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 16 seconds at 2017-06-09 17:08:58.688754825+00:00
++ echo END wait_for_fluentd_to_catch_up took 16 seconds at 2017-06-09 17:08:58.688754825+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-20vz2
++ '[' 0 = 0 ']'
++ python test-viaq-data-model.py test4
++ curl_es_from_kibana logging-kibana-1-20vz2 logging-es project.logging _search message bc03c63b-5593-4717-b304-cc81085a12d5
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer gq0SLvitkDNWYLytx7I6FpfYNPl8zvvHEE_r36Gr3lA' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:bc03c63b-5593-4717-b304-cc81085a12d5'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-20vz2 logging-es-ops .operations _search message 337a6519-d385-4768-a421-599d6dc5c462
++ python test-viaq-data-model.py test4
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer gq0SLvitkDNWYLytx7I6FpfYNPl8zvvHEE_r36Gr3lA' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:337a6519-d385-4768-a421-599d6dc5c462'
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ del_cdm_env_var CDM_EXTRA_KEEP_FIELDS
++ oc get template logging-fluentd-template -o yaml
++ sed '/- name: CDM_EXTRA_KEEP_FIELDS$/,/value:/d'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ add_cdm_env_var_val CDM_EXTRA_KEEP_FIELDS undefined4,undefined5,empty1,undefined3,method,statusCode,type,@timestamp,req,res
+++ mktemp
++ junk=/tmp/tmp.W4kUBEefWm
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.W4kUBEefWm'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.W4kUBEefWm
++ add_cdm_env_var_val CDM_KEEP_EMPTY_FIELDS undefined4,undefined5,empty1,undefined3
+++ mktemp
++ junk=/tmp/tmp.0ezaoNyQq7
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.0ezaoNyQq7'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.0ezaoNyQq7
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-j07sl
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-j07sl
++ '[' -z logging-fluentd-j07sl -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-j07sl
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
pod for component=fluentd not running yet
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-dhzkd
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-dhzkd ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-dhzkd
++ write_and_verify_logs test5 allow_empty
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1497028156
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-09 17:09:16.974250432+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-09 17:09:16.974250432+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-03l0u89m-1-p747h
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
++ '[' -z logging-es-ops-data-master-kq3j3au5-1-8bzmq ']'
+++ uuidgen
++ local uuid_es=caf17305-fa8f-4372-a286-5d9bc60eb0c2
+++ uuidgen
++ local uuid_es_ops=f99e259d-842c-403d-9fbe-051095012a63
++ local expected=1
++ local timeout=300
++ add_test_message caf17305-fa8f-4372-a286-5d9bc60eb0c2
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-20vz2
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/caf17305-fa8f-4372-a286-5d9bc60eb0c2
++ echo added es message caf17305-fa8f-4372-a286-5d9bc60eb0c2
added es message caf17305-fa8f-4372-a286-5d9bc60eb0c2
++ logger -i -p local6.info -t f99e259d-842c-403d-9fbe-051095012a63 f99e259d-842c-403d-9fbe-051095012a63
added es-ops message f99e259d-842c-403d-9fbe-051095012a63
++ echo added es-ops message f99e259d-842c-403d-9fbe-051095012a63
++ local rc=0
++ espod=logging-es-data-master-03l0u89m-1-p747h
++ myproject=project.logging
++ mymessage=caf17305-fa8f-4372-a286-5d9bc60eb0c2
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message caf17305-fa8f-4372-a286-5d9bc60eb0c2
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message caf17305-fa8f-4372-a286-5d9bc60eb0c2
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message caf17305-fa8f-4372-a286-5d9bc60eb0c2
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message caf17305-fa8f-4372-a286-5d9bc60eb0c2
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
+++ shift
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message caf17305-fa8f-4372-a286-5d9bc60eb0c2
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message caf17305-fa8f-4372-a286-5d9bc60eb0c2
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message caf17305-fa8f-4372-a286-5d9bc60eb0c2
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message caf17305-fa8f-4372-a286-5d9bc60eb0c2
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message caf17305-fa8f-4372-a286-5d9bc60eb0c2
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 291 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message caf17305-fa8f-4372-a286-5d9bc60eb0c2
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ local 'endpoint=/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
+++ shift
+++ shift
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 290 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-03l0u89m-1-p747h project.logging _count message caf17305-fa8f-4372-a286-5d9bc60eb0c2
+++ get_count_from_json
+++ curl_es logging-es-data-master-03l0u89m-1-p747h '/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2' --connect-timeout 1
+++ local pod=logging-es-data-master-03l0u89m-1-p747h
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-03l0u89m-1-p747h -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 290 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project logging for caf17305-fa8f-4372-a286-5d9bc60eb0c2
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for caf17305-fa8f-4372-a286-5d9bc60eb0c2
++ espod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
++ myproject=.operations
++ mymessage=f99e259d-842c-403d-9fbe-051095012a63
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-kq3j3au5-1-8bzmq .operations _count systemd.u.SYSLOG_IDENTIFIER f99e259d-842c-403d-9fbe-051095012a63
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-kq3j3au5-1-8bzmq '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:f99e259d-842c-403d-9fbe-051095012a63' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-kq3j3au5-1-8bzmq
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:f99e259d-842c-403d-9fbe-051095012a63'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-kq3j3au5-1-8bzmq -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:f99e259d-842c-403d-9fbe-051095012a63'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for f99e259d-842c-403d-9fbe-051095012a63
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for f99e259d-842c-403d-9fbe-051095012a63
++ '[' -n get_logmessage ']'
++ get_logmessage caf17305-fa8f-4372-a286-5d9bc60eb0c2
++ logmessage=caf17305-fa8f-4372-a286-5d9bc60eb0c2
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 f99e259d-842c-403d-9fbe-051095012a63
++ logmessage2=f99e259d-842c-403d-9fbe-051095012a63
+++ date +%s
++ local endtime=1497028172
+++ expr 1497028172 - 1497028156
+++ date -u --rfc-3339=ns
++ echo END wait_for_fluentd_to_catch_up took 16 seconds at 2017-06-09 17:09:32.876357075+00:00
END wait_for_fluentd_to_catch_up took 16 seconds at 2017-06-09 17:09:32.876357075+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-20vz2
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-20vz2 logging-es project.logging _search message caf17305-fa8f-4372-a286-5d9bc60eb0c2
++ python test-viaq-data-model.py test5 allow_empty
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer gq0SLvitkDNWYLytx7I6FpfYNPl8zvvHEE_r36Gr3lA' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:caf17305-fa8f-4372-a286-5d9bc60eb0c2'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-20vz2 logging-es-ops .operations _search message f99e259d-842c-403d-9fbe-051095012a63
++ python test-viaq-data-model.py test5 allow_empty
++ oc exec logging-kibana-1-20vz2 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer gq0SLvitkDNWYLytx7I6FpfYNPl8zvvHEE_r36Gr3lA' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:f99e259d-842c-403d-9fbe-051095012a63'
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ cleanup
++ remove_test_volume
++ oc get template logging-fluentd-template -o json
++ oc replace -f -
++ python -c 'import json, sys; obj = json.loads(sys.stdin.read()); vm = obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"]; obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] = [xx for xx in vm if xx["name"] != "cdmtest"]; vs = obj["objects"][0]["spec"]["template"]["spec"]["volumes"]; obj["objects"][0]["spec"]["template"]["spec"]["volumes"] = [xx for xx in vs if xx["name"] != "cdmtest"]; print json.dumps(obj, indent=2)'
template "logging-fluentd-template" replaced
++ remove_cdm_env
++ oc get template logging-fluentd-template -o yaml
++ sed '/- name: CDM_/,/value:/d'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.RDolQxko64
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-dhzkd
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-dhzkd
++ '[' -z logging-fluentd-dhzkd -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-dhzkd
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
pod for component=fluentd not running yet
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-fmh08
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-fmh08 ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
SKIPPING reinstall test for now
/data/src/github.com/openshift/origin-aggregated-logging/hack/lib/log/system.sh: line 31:  4631 Terminated              sar -A -o "${binary_logfile}" 1 86400 > /dev/null 2> "${stderr_logfile}"  (wd: /data/src/github.com/openshift/origin-aggregated-logging)
[INFO] [CLEANUP] Beginning cleanup routines...
[INFO] [CLEANUP] Dumping cluster events to /tmp/origin-aggregated-logging/artifacts/events.txt
[INFO] [CLEANUP] Dumping etcd contents to /tmp/origin-aggregated-logging/artifacts/etcd
[WARNING] No compiled `etcdhelper` binary was found. Attempting to build one using:
[WARNING]   $ hack/build-go.sh tools/etcdhelper
++ Building go targets for linux/amd64: tools/etcdhelper
/data/src/github.com/openshift/origin-aggregated-logging/../origin/hack/build-go.sh took 133 seconds
2017-06-09 13:12:18.673371 I | warning: ignoring ServerName for user-provided CA for backwards compatibility is deprecated
[INFO] [CLEANUP] Dumping container logs to /tmp/origin-aggregated-logging/logs/containers
[INFO] [CLEANUP] Truncating log files over 200M
[INFO] [CLEANUP] Stopping docker containers
[INFO] [CLEANUP] Removing docker containers
Error: No such image, container or task: af4778152aa4
json: cannot unmarshal array into Go value of type types.ContainerJSON
Error: No such image, container or task: 76b2d93db69b
json: cannot unmarshal array into Go value of type types.ContainerJSON
Error response from daemon: You cannot remove a running container f91738558655a00ecd7de4497135a14b2f4b868606448f06983e84d1b66bb26f. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container a4b670b064c30c1bcce7c2776d5cc7c927f7e3cb0bc4290eb7d678823d601895. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 2d54c6eff8dd4b518516f2f6197e6b2b7bafa7331f93693034c2eea34810c50d. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 411809a267306f2a925027dc9d7a330d36cce5729d66907c7b03a29fc7c69352. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 2c0e6027dac52c14ae351399fcfbc60675dc7c6c9acf5b7608388f9468bc0b10. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 8fcb129873c9d26bd0c07db7924e946a39058738b575ce2ed4cc85687fdaca53. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container d20e7831b671cb055f288b5c3a7cb174aa80cd805b00dc597a4044405dff295d. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container fb382d6e75dc6f3e92614706e94cec64778f05ae0d3f7356750275af0378fe0f. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 5c6b1b7cc2e8ba148366572119dab27b4ca2faa5f03217608d107db5ca8d643e. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 7489b531745811128aaf9659dbd34d7bdcaa3d994f01e3141f4b09f8c600487a. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container fe009e665b24961c987136d61de85c7215d78ec851fdfa74a85dc782aaec891e. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 15af1cc5e3aac3330feae7dd61bc152c453eda17feed58712dd4404bbe78c222. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 8112e9a370cd1d458fb9b4229091195f0a68dc80a19d934079068e4367dee57b. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 92c24ab653aba274a5c90d14babb7a20c88ca6a23bbb6f3d2a98bef7eacb470e. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container fb1a675e918f0106f9747502faacca78204be1695f0c4862fff95ccdd8642c51. Stop the container before attempting removal or use -f
[INFO] [CLEANUP] Killing child processes
[INFO] [CLEANUP] Pruning etcd data directory
[INFO] /data/src/github.com/openshift/origin-aggregated-logging/logging.sh exited with code 0 after 00h 58m 24s

real	58m24.084s
user	5m32.952s
sys	1m5.280s
Finished GIT_URL=https://github.com/openshift/origin-aggregated-logging GIT_BRANCH=master O_A_L_DIR=/data/src/github.com/openshift/origin-aggregated-logging OS_ROOT=/data/src/github.com/openshift/origin ENABLE_OPS_CLUSTER=true USE_LOCAL_SOURCE=true TEST_PERF=false VERBOSE=1 OS_ANSIBLE_REPO=https://github.com/openshift/openshift-ansible OS_ANSIBLE_BRANCH=master ./logging.sh
***************************************************
==> openshiftdev: Downloading logs
==> openshiftdev: Downloading artifacts from '/var/log/yum.log' to '/var/lib/jenkins/jobs/test-origin-aggregated-logging/workspace@2/origin/artifacts/yum.log'
==> openshiftdev: Downloading artifacts from '/var/log/secure' to '/var/lib/jenkins/jobs/test-origin-aggregated-logging/workspace@2/origin/artifacts/secure'
==> openshiftdev: Downloading artifacts from '/var/log/audit/audit.log' to '/var/lib/jenkins/jobs/test-origin-aggregated-logging/workspace@2/origin/artifacts/audit.log'
==> openshiftdev: Downloading artifacts from '/tmp/origin-aggregated-logging/' to '/var/lib/jenkins/jobs/test-origin-aggregated-logging/workspace@2/origin/artifacts'
+ false
+ false
+ test_pull_requests --mark_test_success --repo origin-aggregated-logging --config /var/lib/jenkins/.test_pull_requests_logging.json
Rate limit remaining: 3509
  Marking SUCCESS for pull request #--repo in repo ''
/usr/share/ruby/net/http/response.rb:119:in `error!': 404 "Not Found" (Net::HTTPServerException)
	from /bin/test_pull_requests:543:in `block in get_comments'
	from /bin/test_pull_requests:535:in `each'
	from /bin/test_pull_requests:535:in `get_comments'
	from /bin/test_pull_requests:1074:in `get_comment_matching_regex'
	from /bin/test_pull_requests:1090:in `get_comment_with_prefix'
	from /bin/test_pull_requests:779:in `mark_test_success'
	from /bin/test_pull_requests:2361:in `<main>'
+ true
[description-setter] Could not determine description.
[PostBuildScript] - Execution post build scripts.
[workspace@2] $ /bin/sh -xe /tmp/hudson8019278742756084700.sh
+ INSTANCE_NAME=origin_logging-rhel7-1655
+ pushd origin
~/jobs/test-origin-aggregated-logging/workspace@2/origin ~/jobs/test-origin-aggregated-logging/workspace@2
+ rc=0
+ '[' -f .vagrant-openshift.json ']'
++ /usr/bin/vagrant ssh -c 'sudo ausearch -m avc'
+ ausearchresult='<no matches>'
+ rc=1
+ '[' '<no matches>' = '<no matches>' ']'
+ rc=0
+ /usr/bin/vagrant destroy -f
==> openshiftdev: Terminating the instance...
==> openshiftdev: Running cleanup tasks for 'shell' provisioner...
+ popd
~/jobs/test-origin-aggregated-logging/workspace@2
+ exit 0
Finished: SUCCESS