SuccessConsole Output

Skipping 576 KB.. Full Log
ift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 260 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 259 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 258 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 257 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 256 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 255 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 254 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 253 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 252 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 251 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 250 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 249 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 248 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 247 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 246 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:9213f45c-6ffc-4f6b-aeb9-d39a20555f3d'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 246 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 9213f45c-6ffc-4f6b-aeb9-d39a20555f3d
++ espod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
++ myproject=.operations
++ mymessage=bfad47a3-b270-4e3d-89a1-e2b5008893fe
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-uuh5lj98-1-vbdzx .operations _count systemd.u.SYSLOG_IDENTIFIER bfad47a3-b270-4e3d-89a1-e2b5008893fe
+++ curl_es logging-es-ops-data-master-uuh5lj98-1-vbdzx '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:bfad47a3-b270-4e3d-89a1-e2b5008893fe' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:bfad47a3-b270-4e3d-89a1-e2b5008893fe'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-uuh5lj98-1-vbdzx -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:bfad47a3-b270-4e3d-89a1-e2b5008893fe'
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for bfad47a3-b270-4e3d-89a1-e2b5008893fe
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for bfad47a3-b270-4e3d-89a1-e2b5008893fe
++ '[' -n '' ']'
++ '[' -n '' ']'
+++ date +%s
++ local endtime=1496949036
+++ expr 1496949036 - 1496948960
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 76 seconds at 2017-06-08 19:10:36.158904000+00:00
++ echo END wait_for_fluentd_to_catch_up took 76 seconds at 2017-06-08 19:10:36.158904000+00:00
++ return 0
++ return 0
++ cleanup
++ cleanup_forward
++ oc label node --all logging-infra-fluentd-
node "172.18.13.130" labeled
++ wait_for_pod_ACTION stop logging-fluentd-bjgkz
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-bjgkz
++ '[' -z logging-fluentd-bjgkz -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-bjgkz
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc delete daemonset/logging-forward-fluentd
daemonset "logging-forward-fluentd" deleted
+++ oc get configmap/logging-fluentd -o yaml
+++ grep '<match \*\*>'
++ '[' -n '      <match **>' ']'
++ oc get configmap/logging-fluentd -o yaml
++ sed -e '/<match \*\*>/ d' -e '/@include configs\.d\/user\/secure-forward\.conf/ d' -e '/<\/match>/ d'
++ oc replace -f -
configmap "logging-fluentd" replaced
++ oc patch configmap/logging-fluentd --type=json --patch '[{ "op": "replace", "path": "/data/secure-forward.conf", "value": "\
# @type secure_forward\n\
# self_hostname forwarding-${HOSTNAME}\n\
# shared_key aggregated_logging_ci_testing\n\
#  secure no\n\
#  <server>\n\
#   host ${FLUENTD_FORWARD}\n\
#   port 24284\n\
#  </server>"}]'
configmap "logging-fluentd" patched
++ oc label node --all logging-infra-fluentd=true
node "172.18.13.130" labeled
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
pod for component=fluentd not running yet
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-gf6qq
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-gf6qq ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-gf6qq
++ oc get events -o yaml
++ write_and_verify_logs 1
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up '' ''
+++ date +%s
++ local starttime=1496949051
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-08 19:10:51.708420831+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-08 19:10:51.708420831+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
++ '[' -z logging-es-ops-data-master-uuh5lj98-1-vbdzx ']'
+++ uuidgen
++ local uuid_es=1a4f6d69-b6ec-468f-9b22-2ae421257441
+++ uuidgen
++ local uuid_es_ops=6002da4c-c2dc-4077-b466-437d1d65dcb8
++ local expected=1
++ local timeout=300
++ add_test_message 1a4f6d69-b6ec-468f-9b22-2ae421257441
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-86619
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/1a4f6d69-b6ec-468f-9b22-2ae421257441
++ echo added es message 1a4f6d69-b6ec-468f-9b22-2ae421257441
added es message 1a4f6d69-b6ec-468f-9b22-2ae421257441
++ logger -i -p local6.info -t 6002da4c-c2dc-4077-b466-437d1d65dcb8 6002da4c-c2dc-4077-b466-437d1d65dcb8
added es-ops message 6002da4c-c2dc-4077-b466-437d1d65dcb8
++ echo added es-ops message 6002da4c-c2dc-4077-b466-437d1d65dcb8
++ local rc=0
++ espod=logging-es-data-master-zbfzxdm0-1-w1lc1
++ myproject=project.logging
++ mymessage=1a4f6d69-b6ec-468f-9b22-2ae421257441
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 1a4f6d69-b6ec-468f-9b22-2ae421257441
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 1a4f6d69-b6ec-468f-9b22-2ae421257441
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 1a4f6d69-b6ec-468f-9b22-2ae421257441
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 1a4f6d69-b6ec-468f-9b22-2ae421257441
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 1a4f6d69-b6ec-468f-9b22-2ae421257441
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 1a4f6d69-b6ec-468f-9b22-2ae421257441
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 1a4f6d69-b6ec-468f-9b22-2ae421257441
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 1a4f6d69-b6ec-468f-9b22-2ae421257441
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 1a4f6d69-b6ec-468f-9b22-2ae421257441
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 291 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 1a4f6d69-b6ec-468f-9b22-2ae421257441
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 290 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 1a4f6d69-b6ec-468f-9b22-2ae421257441
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 289 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 1a4f6d69-b6ec-468f-9b22-2ae421257441
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 288 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 1a4f6d69-b6ec-468f-9b22-2ae421257441
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 287 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 1a4f6d69-b6ec-468f-9b22-2ae421257441
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:1a4f6d69-b6ec-468f-9b22-2ae421257441'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 287 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 1a4f6d69-b6ec-468f-9b22-2ae421257441
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 1a4f6d69-b6ec-468f-9b22-2ae421257441
++ espod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
++ myproject=.operations
++ mymessage=6002da4c-c2dc-4077-b466-437d1d65dcb8
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-uuh5lj98-1-vbdzx .operations _count systemd.u.SYSLOG_IDENTIFIER 6002da4c-c2dc-4077-b466-437d1d65dcb8
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-uuh5lj98-1-vbdzx '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:6002da4c-c2dc-4077-b466-437d1d65dcb8' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:6002da4c-c2dc-4077-b466-437d1d65dcb8'
+++ shift
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-uuh5lj98-1-vbdzx -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:6002da4c-c2dc-4077-b466-437d1d65dcb8'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 6002da4c-c2dc-4077-b466-437d1d65dcb8
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 6002da4c-c2dc-4077-b466-437d1d65dcb8
++ '[' -n '' ']'
++ '[' -n '' ']'
+++ date +%s
++ local endtime=1496949071
+++ expr 1496949071 - 1496949051
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 20 seconds at 2017-06-08 19:11:11.312936165+00:00
++ echo END wait_for_fluentd_to_catch_up took 20 seconds at 2017-06-08 19:11:11.312936165+00:00
++ return 0
++ return 0
++ cleanup
++ cleanup_forward
++ oc label node --all logging-infra-fluentd-
node "172.18.13.130" labeled
++ wait_for_pod_ACTION stop logging-fluentd-gf6qq
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-gf6qq
++ '[' -z logging-fluentd-gf6qq -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-gf6qq
++ '[' -n 1 ']'
pod logging-fluentd-gf6qq still running
++ echo pod logging-fluentd-gf6qq still running
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' stop = start ']'
++ '[' 110 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-gf6qq
++ '[' stop = start ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
++ oc delete daemonset/logging-forward-fluentd
Error from server (NotFound): daemonsets.extensions "logging-forward-fluentd" not found
++ :
+++ oc get configmap/logging-fluentd -o yaml
+++ grep '<match \*\*>'
++ '[' -n '' ']'
++ oc patch configmap/logging-fluentd --type=json --patch '[{ "op": "replace", "path": "/data/secure-forward.conf", "value": "\
# @type secure_forward\n\
# self_hostname forwarding-${HOSTNAME}\n\
# shared_key aggregated_logging_ci_testing\n\
#  secure no\n\
#  <server>\n\
#   host ${FLUENTD_FORWARD}\n\
#   port 24284\n\
#  </server>"}]'
configmap "logging-fluentd" patched
++ oc label node --all logging-infra-fluentd=true
node "172.18.13.130" labeled
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
pod for component=fluentd not running yet
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-m6r5d
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-m6r5d ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-m6r5d
++ oc get events -o yaml
running test test-json-parsing.sh
++ set -o nounset
++ set -o pipefail
++ type get_running_pod
++ ARTIFACT_DIR=/tmp/origin-aggregated-logging/artifacts
++ '[' '!' -d /tmp/origin-aggregated-logging/artifacts ']'
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ es_pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ uuidgen
++ uuid_es=d3a40876-f155-414d-88d0-9b6a08499109
++ echo Adding test message d3a40876-f155-414d-88d0-9b6a08499109 to Kibana . . .
Adding test message d3a40876-f155-414d-88d0-9b6a08499109 to Kibana . . .
++ add_test_message d3a40876-f155-414d-88d0-9b6a08499109
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-86619
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/d3a40876-f155-414d-88d0-9b6a08499109
++ rc=0
++ timeout=600
++ echo Waiting 600 seconds for d3a40876-f155-414d-88d0-9b6a08499109 to show up in Elasticsearch . . .
Waiting 600 seconds for d3a40876-f155-414d-88d0-9b6a08499109 to show up in Elasticsearch . . .
++ espod=logging-es-data-master-zbfzxdm0-1-w1lc1
++ myproject=project.logging.
++ mymessage=d3a40876-f155-414d-88d0-9b6a08499109
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 600
++ let ii=600
++ local interval=1
++ '[' 600 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging. _count message d3a40876-f155-414d-88d0-9b6a08499109
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 599 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging. _count message d3a40876-f155-414d-88d0-9b6a08499109
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109' --connect-timeout 1
+++ get_count_from_json
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 598 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging. _count message d3a40876-f155-414d-88d0-9b6a08499109
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 597 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging. _count message d3a40876-f155-414d-88d0-9b6a08499109
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 596 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging. _count message d3a40876-f155-414d-88d0-9b6a08499109
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 595 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging. _count message d3a40876-f155-414d-88d0-9b6a08499109
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 594 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging. _count message d3a40876-f155-414d-88d0-9b6a08499109
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 593 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging. _count message d3a40876-f155-414d-88d0-9b6a08499109
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 592 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging. _count message d3a40876-f155-414d-88d0-9b6a08499109
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_count?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 592 -le 0 ']'
++ return 0
++ echo good - ./logging.sh: found 1 record project logging for d3a40876-f155-414d-88d0-9b6a08499109
good - ./logging.sh: found 1 record project logging for d3a40876-f155-414d-88d0-9b6a08499109
Testing if record is in correct format . . .
++ echo Testing if record is in correct format . . .
++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging. _search message d3a40876-f155-414d-88d0-9b6a08499109
++ python test-json-parsing.py d3a40876-f155-414d-88d0-9b6a08499109
++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging.*/_search?q=message:d3a40876-f155-414d-88d0-9b6a08499109' --connect-timeout 1
++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
++ local 'endpoint=/project.logging.*/_search?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
++ shift
++ shift
++ args=("${@:-}")
++ local args
++ local secret_dir=/etc/elasticsearch/secret/
++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging.*/_search?q=message:d3a40876-f155-414d-88d0-9b6a08499109'
Success: record contains all of the expected fields/values
Success: ./logging.sh passed
++ echo Success: ./logging.sh passed
++ exit 0
running test test-mux.sh
++ set -o nounset
++ set -o pipefail
++ type get_running_pod
++ '[' false == false -o false == false ']'
Skipping -- This test requires both USE_MUX_CLIENT and MUX_ALLOW_EXTERNAL are true.
++ echo 'Skipping -- This test requires both USE_MUX_CLIENT and MUX_ALLOW_EXTERNAL are true.'
++ exit 0
SKIPPING upgrade test for now
running test test-viaq-data-model.sh
++ set -o nounset
++ set -o pipefail
++ type get_running_pod
++ [[ 1 -ne 1 ]]
++ [[ true = \f\a\l\s\e ]]
++ CLUSTER=true
++ ops=-ops
++ INDEX_PREFIX=
++ PROJ_PREFIX=project.
++ ARTIFACT_DIR=/tmp/origin-aggregated-logging/artifacts
++ '[' '!' -d /tmp/origin-aggregated-logging/artifacts ']'
++ get_test_user_token
++ local current_project
+++ oc project -q
++ current_project=logging
++ oc login --username=admin --password=admin
+++ oc whoami -t
++ test_token=xQLLRNSoVy0CkrprwHB0cNYkyG-yXiqnF78837UFjXk
+++ oc whoami
++ test_name=admin
++ test_ip=127.0.0.1
++ oc login --username=system:admin
++ oc project logging
++ TEST_DIVIDER=------------------------------------------
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-m6r5d
++ remove_test_volume
++ oc get template logging-fluentd-template -o json
++ python -c 'import json, sys; obj = json.loads(sys.stdin.read()); vm = obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"]; obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] = [xx for xx in vm if xx["name"] != "cdmtest"]; vs = obj["objects"][0]["spec"]["template"]["spec"]["volumes"]; obj["objects"][0]["spec"]["template"]["spec"]["volumes"] = [xx for xx in vs if xx["name"] != "cdmtest"]; print json.dumps(obj, indent=2)'
++ oc replace -f -
template "logging-fluentd-template" replaced
+++ mktemp
++ cfg=/tmp/tmp.s1tQLSzDYA
++ cat
++ add_test_volume /tmp/tmp.s1tQLSzDYA
++ oc get template logging-fluentd-template -o json
++ python -c 'import json, sys; obj = json.loads(sys.stdin.read()); obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"].append({"name": "cdmtest", "mountPath": "/etc/fluent/configs.d/openshift/filter-pre-cdm-test.conf", "readOnly": True}); obj["objects"][0]["spec"]["template"]["spec"]["volumes"].append({"name": "cdmtest", "hostPath": {"path": "/tmp/tmp.s1tQLSzDYA"}}); print json.dumps(obj, indent=2)'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ trap cleanup INT TERM EXIT
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-m6r5d
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-m6r5d
++ '[' -z logging-fluentd-m6r5d -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-m6r5d
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc create -f -
++ oc process logging-fluentd-template
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-4vbqm
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-4vbqm ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-4vbqm
++ keep_fields=method,statusCode,type,@timestamp,req,res
++ write_and_verify_logs test1
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1496949127
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-08 19:12:07.873790246+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-08 19:12:07.873790246+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
++ '[' -z logging-es-ops-data-master-uuh5lj98-1-vbdzx ']'
+++ uuidgen
++ local uuid_es=f4342d92-8c3f-4e11-a43c-3fcfede2f490
+++ uuidgen
++ local uuid_es_ops=e600cf03-1f52-475a-a5af-36f8d42c7ff9
++ local expected=1
++ local timeout=300
++ add_test_message f4342d92-8c3f-4e11-a43c-3fcfede2f490
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-86619
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/f4342d92-8c3f-4e11-a43c-3fcfede2f490
added es message f4342d92-8c3f-4e11-a43c-3fcfede2f490
++ echo added es message f4342d92-8c3f-4e11-a43c-3fcfede2f490
++ logger -i -p local6.info -t e600cf03-1f52-475a-a5af-36f8d42c7ff9 e600cf03-1f52-475a-a5af-36f8d42c7ff9
added es-ops message e600cf03-1f52-475a-a5af-36f8d42c7ff9
++ echo added es-ops message e600cf03-1f52-475a-a5af-36f8d42c7ff9
++ local rc=0
++ espod=logging-es-data-master-zbfzxdm0-1-w1lc1
++ myproject=project.logging
++ mymessage=f4342d92-8c3f-4e11-a43c-3fcfede2f490
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message f4342d92-8c3f-4e11-a43c-3fcfede2f490
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message f4342d92-8c3f-4e11-a43c-3fcfede2f490
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message f4342d92-8c3f-4e11-a43c-3fcfede2f490
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message f4342d92-8c3f-4e11-a43c-3fcfede2f490
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message f4342d92-8c3f-4e11-a43c-3fcfede2f490
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message f4342d92-8c3f-4e11-a43c-3fcfede2f490
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message f4342d92-8c3f-4e11-a43c-3fcfede2f490
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message f4342d92-8c3f-4e11-a43c-3fcfede2f490
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message f4342d92-8c3f-4e11-a43c-3fcfede2f490
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 292 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project logging for f4342d92-8c3f-4e11-a43c-3fcfede2f490
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for f4342d92-8c3f-4e11-a43c-3fcfede2f490
++ espod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
++ myproject=.operations
++ mymessage=e600cf03-1f52-475a-a5af-36f8d42c7ff9
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-uuh5lj98-1-vbdzx .operations _count systemd.u.SYSLOG_IDENTIFIER e600cf03-1f52-475a-a5af-36f8d42c7ff9
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-uuh5lj98-1-vbdzx '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:e600cf03-1f52-475a-a5af-36f8d42c7ff9' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:e600cf03-1f52-475a-a5af-36f8d42c7ff9'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-uuh5lj98-1-vbdzx -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:e600cf03-1f52-475a-a5af-36f8d42c7ff9'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for e600cf03-1f52-475a-a5af-36f8d42c7ff9
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for e600cf03-1f52-475a-a5af-36f8d42c7ff9
++ '[' -n get_logmessage ']'
++ get_logmessage f4342d92-8c3f-4e11-a43c-3fcfede2f490
++ logmessage=f4342d92-8c3f-4e11-a43c-3fcfede2f490
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 e600cf03-1f52-475a-a5af-36f8d42c7ff9
++ logmessage2=e600cf03-1f52-475a-a5af-36f8d42c7ff9
+++ date +%s
++ local endtime=1496949140
+++ expr 1496949140 - 1496949127
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 13 seconds at 2017-06-08 19:12:20.983479104+00:00
++ echo END wait_for_fluentd_to_catch_up took 13 seconds at 2017-06-08 19:12:20.983479104+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-86619
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-86619 logging-es project.logging _search message f4342d92-8c3f-4e11-a43c-3fcfede2f490
++ python test-viaq-data-model.py test1
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer xQLLRNSoVy0CkrprwHB0cNYkyG-yXiqnF78837UFjXk' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:f4342d92-8c3f-4e11-a43c-3fcfede2f490'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-86619 logging-es-ops .operations _search message e600cf03-1f52-475a-a5af-36f8d42c7ff9
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer xQLLRNSoVy0CkrprwHB0cNYkyG-yXiqnF78837UFjXk' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:e600cf03-1f52-475a-a5af-36f8d42c7ff9'
++ python test-viaq-data-model.py test1
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ add_cdm_env_var_val CDM_USE_UNDEFINED '"true"'
+++ mktemp
++ junk=/tmp/tmp.sxJqFOEl0I
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.sxJqFOEl0I'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.sxJqFOEl0I
++ add_cdm_env_var_val CDM_EXTRA_KEEP_FIELDS method,statusCode,type,@timestamp,req,res
+++ mktemp
++ junk=/tmp/tmp.X495rwGa8T
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.X495rwGa8T'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.X495rwGa8T
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-4vbqm
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-4vbqm
++ '[' -z logging-fluentd-4vbqm -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-4vbqm
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-9fjpr
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-9fjpr ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-9fjpr
++ write_and_verify_logs test2
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1496949159
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-08 19:12:39.242527102+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-08 19:12:39.242527102+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
++ '[' -z logging-es-ops-data-master-uuh5lj98-1-vbdzx ']'
+++ uuidgen
++ local uuid_es=6335e97d-fc19-40b7-ab68-175d931e51e7
+++ uuidgen
++ local uuid_es_ops=f6844094-423d-471d-a3d0-d8ec6911fe24
++ local expected=1
++ local timeout=300
++ add_test_message 6335e97d-fc19-40b7-ab68-175d931e51e7
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-86619
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/6335e97d-fc19-40b7-ab68-175d931e51e7
added es message 6335e97d-fc19-40b7-ab68-175d931e51e7
++ echo added es message 6335e97d-fc19-40b7-ab68-175d931e51e7
++ logger -i -p local6.info -t f6844094-423d-471d-a3d0-d8ec6911fe24 f6844094-423d-471d-a3d0-d8ec6911fe24
added es-ops message f6844094-423d-471d-a3d0-d8ec6911fe24
++ echo added es-ops message f6844094-423d-471d-a3d0-d8ec6911fe24
++ local rc=0
++ espod=logging-es-data-master-zbfzxdm0-1-w1lc1
++ myproject=project.logging
++ mymessage=6335e97d-fc19-40b7-ab68-175d931e51e7
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 6335e97d-fc19-40b7-ab68-175d931e51e7
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 6335e97d-fc19-40b7-ab68-175d931e51e7
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 6335e97d-fc19-40b7-ab68-175d931e51e7
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 6335e97d-fc19-40b7-ab68-175d931e51e7
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 6335e97d-fc19-40b7-ab68-175d931e51e7
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 6335e97d-fc19-40b7-ab68-175d931e51e7
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 6335e97d-fc19-40b7-ab68-175d931e51e7
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 6335e97d-fc19-40b7-ab68-175d931e51e7
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 293 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 6335e97d-fc19-40b7-ab68-175d931e51e7
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 6335e97d-fc19-40b7-ab68-175d931e51e7
++ espod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
++ myproject=.operations
++ mymessage=f6844094-423d-471d-a3d0-d8ec6911fe24
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-uuh5lj98-1-vbdzx .operations _count systemd.u.SYSLOG_IDENTIFIER f6844094-423d-471d-a3d0-d8ec6911fe24
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-uuh5lj98-1-vbdzx '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:f6844094-423d-471d-a3d0-d8ec6911fe24' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:f6844094-423d-471d-a3d0-d8ec6911fe24'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-uuh5lj98-1-vbdzx -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:f6844094-423d-471d-a3d0-d8ec6911fe24'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for f6844094-423d-471d-a3d0-d8ec6911fe24
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for f6844094-423d-471d-a3d0-d8ec6911fe24
++ '[' -n get_logmessage ']'
++ get_logmessage 6335e97d-fc19-40b7-ab68-175d931e51e7
++ logmessage=6335e97d-fc19-40b7-ab68-175d931e51e7
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 f6844094-423d-471d-a3d0-d8ec6911fe24
++ logmessage2=f6844094-423d-471d-a3d0-d8ec6911fe24
+++ date +%s
++ local endtime=1496949170
+++ expr 1496949170 - 1496949159
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 11 seconds at 2017-06-08 19:12:50.897975005+00:00
++ echo END wait_for_fluentd_to_catch_up took 11 seconds at 2017-06-08 19:12:50.897975005+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-86619
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-86619 logging-es project.logging _search message 6335e97d-fc19-40b7-ab68-175d931e51e7
++ python test-viaq-data-model.py test2
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer xQLLRNSoVy0CkrprwHB0cNYkyG-yXiqnF78837UFjXk' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:6335e97d-fc19-40b7-ab68-175d931e51e7'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-86619 logging-es-ops .operations _search message f6844094-423d-471d-a3d0-d8ec6911fe24
++ python test-viaq-data-model.py test2
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer xQLLRNSoVy0CkrprwHB0cNYkyG-yXiqnF78837UFjXk' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:f6844094-423d-471d-a3d0-d8ec6911fe24'
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ del_cdm_env_var CDM_EXTRA_KEEP_FIELDS
++ oc get template logging-fluentd-template -o yaml
++ sed '/- name: CDM_EXTRA_KEEP_FIELDS$/,/value:/d'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ add_cdm_env_var_val CDM_EXTRA_KEEP_FIELDS undefined4,undefined5,method,statusCode,type,@timestamp,req,res
+++ mktemp
++ junk=/tmp/tmp.7ZXv4XqF7G
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.7ZXv4XqF7G'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.7ZXv4XqF7G
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-9fjpr
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-9fjpr
++ '[' -z logging-fluentd-9fjpr -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-9fjpr
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
pod for component=fluentd not running yet
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-gs8kw
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-gs8kw ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-gs8kw
++ write_and_verify_logs test3
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1496949197
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-08 19:13:17.928914762+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-08 19:13:17.928914762+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
++ '[' -z logging-es-ops-data-master-uuh5lj98-1-vbdzx ']'
+++ uuidgen
++ local uuid_es=0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
+++ uuidgen
++ local uuid_es_ops=72b17de1-2fcd-40fc-9222-5aaea1f0802e
++ local expected=1
++ local timeout=300
++ add_test_message 0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-86619
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
++ echo added es message 0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
added es message 0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
++ logger -i -p local6.info -t 72b17de1-2fcd-40fc-9222-5aaea1f0802e 72b17de1-2fcd-40fc-9222-5aaea1f0802e
added es-ops message 72b17de1-2fcd-40fc-9222-5aaea1f0802e
++ echo added es-ops message 72b17de1-2fcd-40fc-9222-5aaea1f0802e
++ local rc=0
++ espod=logging-es-data-master-zbfzxdm0-1-w1lc1
++ myproject=project.logging
++ mymessage=0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ get_count_from_json
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 292 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
++ espod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
++ myproject=.operations
++ mymessage=72b17de1-2fcd-40fc-9222-5aaea1f0802e
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-uuh5lj98-1-vbdzx .operations _count systemd.u.SYSLOG_IDENTIFIER 72b17de1-2fcd-40fc-9222-5aaea1f0802e
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-uuh5lj98-1-vbdzx '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:72b17de1-2fcd-40fc-9222-5aaea1f0802e' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:72b17de1-2fcd-40fc-9222-5aaea1f0802e'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-uuh5lj98-1-vbdzx -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:72b17de1-2fcd-40fc-9222-5aaea1f0802e'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-uuh5lj98-1-vbdzx .operations _count systemd.u.SYSLOG_IDENTIFIER 72b17de1-2fcd-40fc-9222-5aaea1f0802e
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-ops-data-master-uuh5lj98-1-vbdzx '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:72b17de1-2fcd-40fc-9222-5aaea1f0802e' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:72b17de1-2fcd-40fc-9222-5aaea1f0802e'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-uuh5lj98-1-vbdzx -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:72b17de1-2fcd-40fc-9222-5aaea1f0802e'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-uuh5lj98-1-vbdzx .operations _count systemd.u.SYSLOG_IDENTIFIER 72b17de1-2fcd-40fc-9222-5aaea1f0802e
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-uuh5lj98-1-vbdzx '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:72b17de1-2fcd-40fc-9222-5aaea1f0802e' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:72b17de1-2fcd-40fc-9222-5aaea1f0802e'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-uuh5lj98-1-vbdzx -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:72b17de1-2fcd-40fc-9222-5aaea1f0802e'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-uuh5lj98-1-vbdzx .operations _count systemd.u.SYSLOG_IDENTIFIER 72b17de1-2fcd-40fc-9222-5aaea1f0802e
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-uuh5lj98-1-vbdzx '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:72b17de1-2fcd-40fc-9222-5aaea1f0802e' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:72b17de1-2fcd-40fc-9222-5aaea1f0802e'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-uuh5lj98-1-vbdzx -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:72b17de1-2fcd-40fc-9222-5aaea1f0802e'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-uuh5lj98-1-vbdzx .operations _count systemd.u.SYSLOG_IDENTIFIER 72b17de1-2fcd-40fc-9222-5aaea1f0802e
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-uuh5lj98-1-vbdzx '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:72b17de1-2fcd-40fc-9222-5aaea1f0802e' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:72b17de1-2fcd-40fc-9222-5aaea1f0802e'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-uuh5lj98-1-vbdzx -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:72b17de1-2fcd-40fc-9222-5aaea1f0802e'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 296 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 72b17de1-2fcd-40fc-9222-5aaea1f0802e
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 72b17de1-2fcd-40fc-9222-5aaea1f0802e
++ '[' -n get_logmessage ']'
++ get_logmessage 0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
++ logmessage=0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 72b17de1-2fcd-40fc-9222-5aaea1f0802e
++ logmessage2=72b17de1-2fcd-40fc-9222-5aaea1f0802e
+++ date +%s
++ local endtime=1496949216
+++ expr 1496949216 - 1496949197
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 19 seconds at 2017-06-08 19:13:36.307048616+00:00
++ echo END wait_for_fluentd_to_catch_up took 19 seconds at 2017-06-08 19:13:36.307048616+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-86619
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-86619 logging-es project.logging _search message 0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6
++ python test-viaq-data-model.py test3
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer xQLLRNSoVy0CkrprwHB0cNYkyG-yXiqnF78837UFjXk' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:0b3d7f15-a70c-4763-ac20-2b8ff2a6cbd6'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-86619 logging-es-ops .operations _search message 72b17de1-2fcd-40fc-9222-5aaea1f0802e
++ python test-viaq-data-model.py test3
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer xQLLRNSoVy0CkrprwHB0cNYkyG-yXiqnF78837UFjXk' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:72b17de1-2fcd-40fc-9222-5aaea1f0802e'
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ add_cdm_env_var_val CDM_UNDEFINED_NAME myname
+++ mktemp
++ junk=/tmp/tmp.KSEvDD2VIO
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.KSEvDD2VIO'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.KSEvDD2VIO
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-gs8kw
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-gs8kw
++ '[' -z logging-fluentd-gs8kw -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-gs8kw
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
pod for component=fluentd not running yet
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-jg5b0
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-jg5b0 ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-jg5b0
++ write_and_verify_logs test4
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1496949237
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-08 19:13:57.929367686+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-08 19:13:57.929367686+00:00
+++ get_running_pod es
+++ oc get pods -l component=es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
++ '[' -z logging-es-ops-data-master-uuh5lj98-1-vbdzx ']'
+++ uuidgen
++ local uuid_es=b4d2df2b-d3ba-421c-bba9-4403502d4844
+++ uuidgen
++ local uuid_es_ops=43656356-532b-4c62-ae27-959bfb90499f
++ local expected=1
++ local timeout=300
++ add_test_message b4d2df2b-d3ba-421c-bba9-4403502d4844
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-86619
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/b4d2df2b-d3ba-421c-bba9-4403502d4844
added es message b4d2df2b-d3ba-421c-bba9-4403502d4844
++ echo added es message b4d2df2b-d3ba-421c-bba9-4403502d4844
++ logger -i -p local6.info -t 43656356-532b-4c62-ae27-959bfb90499f 43656356-532b-4c62-ae27-959bfb90499f
added es-ops message 43656356-532b-4c62-ae27-959bfb90499f
++ echo added es-ops message 43656356-532b-4c62-ae27-959bfb90499f
++ local rc=0
++ espod=logging-es-data-master-zbfzxdm0-1-w1lc1
++ myproject=project.logging
++ mymessage=b4d2df2b-d3ba-421c-bba9-4403502d4844
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message b4d2df2b-d3ba-421c-bba9-4403502d4844
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message b4d2df2b-d3ba-421c-bba9-4403502d4844
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ get_count_from_json
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message b4d2df2b-d3ba-421c-bba9-4403502d4844
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message b4d2df2b-d3ba-421c-bba9-4403502d4844
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message b4d2df2b-d3ba-421c-bba9-4403502d4844
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message b4d2df2b-d3ba-421c-bba9-4403502d4844
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message b4d2df2b-d3ba-421c-bba9-4403502d4844
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message b4d2df2b-d3ba-421c-bba9-4403502d4844
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
+++ get_count_from_json
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message b4d2df2b-d3ba-421c-bba9-4403502d4844
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 292 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project logging for b4d2df2b-d3ba-421c-bba9-4403502d4844
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for b4d2df2b-d3ba-421c-bba9-4403502d4844
++ espod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
++ myproject=.operations
++ mymessage=43656356-532b-4c62-ae27-959bfb90499f
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-uuh5lj98-1-vbdzx .operations _count systemd.u.SYSLOG_IDENTIFIER 43656356-532b-4c62-ae27-959bfb90499f
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-uuh5lj98-1-vbdzx '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:43656356-532b-4c62-ae27-959bfb90499f' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:43656356-532b-4c62-ae27-959bfb90499f'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-uuh5lj98-1-vbdzx -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:43656356-532b-4c62-ae27-959bfb90499f'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 43656356-532b-4c62-ae27-959bfb90499f
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for 43656356-532b-4c62-ae27-959bfb90499f
++ '[' -n get_logmessage ']'
++ get_logmessage b4d2df2b-d3ba-421c-bba9-4403502d4844
++ logmessage=b4d2df2b-d3ba-421c-bba9-4403502d4844
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 43656356-532b-4c62-ae27-959bfb90499f
++ logmessage2=43656356-532b-4c62-ae27-959bfb90499f
+++ date +%s
++ local endtime=1496949250
+++ expr 1496949250 - 1496949237
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 13 seconds at 2017-06-08 19:14:10.872684323+00:00
++ echo END wait_for_fluentd_to_catch_up took 13 seconds at 2017-06-08 19:14:10.872684323+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-86619
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-86619 logging-es project.logging _search message b4d2df2b-d3ba-421c-bba9-4403502d4844
++ python test-viaq-data-model.py test4
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer xQLLRNSoVy0CkrprwHB0cNYkyG-yXiqnF78837UFjXk' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:b4d2df2b-d3ba-421c-bba9-4403502d4844'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-86619 logging-es-ops .operations _search message 43656356-532b-4c62-ae27-959bfb90499f
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer xQLLRNSoVy0CkrprwHB0cNYkyG-yXiqnF78837UFjXk' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:43656356-532b-4c62-ae27-959bfb90499f'
++ python test-viaq-data-model.py test4
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ del_cdm_env_var CDM_EXTRA_KEEP_FIELDS
++ oc get template logging-fluentd-template -o yaml
++ sed '/- name: CDM_EXTRA_KEEP_FIELDS$/,/value:/d'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ add_cdm_env_var_val CDM_EXTRA_KEEP_FIELDS undefined4,undefined5,empty1,undefined3,method,statusCode,type,@timestamp,req,res
+++ mktemp
++ junk=/tmp/tmp.95PF3BJQqW
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.95PF3BJQqW'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.95PF3BJQqW
++ add_cdm_env_var_val CDM_KEEP_EMPTY_FIELDS undefined4,undefined5,empty1,undefined3
+++ mktemp
++ junk=/tmp/tmp.RMKkqHt5Kc
++ cat
++ oc get template logging-fluentd-template -o yaml
++ sed '/env:/r /tmp/tmp.RMKkqHt5Kc'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.RMKkqHt5Kc
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-jg5b0
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-jg5b0
++ '[' -z logging-fluentd-jg5b0 -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-jg5b0
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
++ echo pod for component=fluentd not running yet
pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-6mtlj
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-6mtlj ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ fpod=logging-fluentd-6mtlj
++ write_and_verify_logs test5 allow_empty
++ expected=1
++ rc=0
++ wait_for_fluentd_to_catch_up get_logmessage get_logmessage2
+++ date +%s
++ local starttime=1496949278
+++ date -u --rfc-3339=ns
START wait_for_fluentd_to_catch_up at 2017-06-08 19:14:38.186509886+00:00
++ echo START wait_for_fluentd_to_catch_up at 2017-06-08 19:14:38.186509886+00:00
+++ get_running_pod es
+++ awk -v sel=es '$1 ~ sel && $3 == "Running" {print $1}'
+++ oc get pods -l component=es
++ local es_pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ get_running_pod es-ops
+++ oc get pods -l component=es-ops
+++ awk -v sel=es-ops '$1 ~ sel && $3 == "Running" {print $1}'
++ local es_ops_pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
++ '[' -z logging-es-ops-data-master-uuh5lj98-1-vbdzx ']'
+++ uuidgen
++ local uuid_es=869f7d82-5866-4867-a749-53e06dfe24d5
+++ uuidgen
++ local uuid_es_ops=a12e5b5d-afe3-46c5-b16d-315db92670b8
++ local expected=1
++ local timeout=300
++ add_test_message 869f7d82-5866-4867-a749-53e06dfe24d5
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ local kib_pod=logging-kibana-1-86619
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s http://localhost:5601/869f7d82-5866-4867-a749-53e06dfe24d5
++ echo added es message 869f7d82-5866-4867-a749-53e06dfe24d5
added es message 869f7d82-5866-4867-a749-53e06dfe24d5
++ logger -i -p local6.info -t a12e5b5d-afe3-46c5-b16d-315db92670b8 a12e5b5d-afe3-46c5-b16d-315db92670b8
added es-ops message a12e5b5d-afe3-46c5-b16d-315db92670b8
++ echo added es-ops message a12e5b5d-afe3-46c5-b16d-315db92670b8
++ local rc=0
++ espod=logging-es-data-master-zbfzxdm0-1-w1lc1
++ myproject=project.logging
++ mymessage=869f7d82-5866-4867-a749-53e06dfe24d5
++ expected=1
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 869f7d82-5866-4867-a749-53e06dfe24d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 299 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 869f7d82-5866-4867-a749-53e06dfe24d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 298 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 869f7d82-5866-4867-a749-53e06dfe24d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 297 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 869f7d82-5866-4867-a749-53e06dfe24d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 296 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 869f7d82-5866-4867-a749-53e06dfe24d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 295 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 869f7d82-5866-4867-a749-53e06dfe24d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 294 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 869f7d82-5866-4867-a749-53e06dfe24d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5' --connect-timeout 1
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 293 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 869f7d82-5866-4867-a749-53e06dfe24d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
++ local nrecs=0
++ test 0 = 1
++ sleep 1
++ let ii=ii-1
++ '[' 292 -gt 0 ']'
++ test_count_expected
++ myfield=message
+++ query_es_from_es logging-es-data-master-zbfzxdm0-1-w1lc1 project.logging _count message 869f7d82-5866-4867-a749-53e06dfe24d5
+++ get_count_from_json
+++ curl_es logging-es-data-master-zbfzxdm0-1-w1lc1 '/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5' --connect-timeout 1
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local pod=logging-es-data-master-zbfzxdm0-1-w1lc1
+++ local 'endpoint=/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-data-master-zbfzxdm0-1-w1lc1 -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/project.logging*/_count?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 292 -le 0 ']'
++ return 0
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project logging for 869f7d82-5866-4867-a749-53e06dfe24d5
good - wait_for_fluentd_to_catch_up: found 1 record project logging for 869f7d82-5866-4867-a749-53e06dfe24d5
++ espod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
++ myproject=.operations
++ mymessage=a12e5b5d-afe3-46c5-b16d-315db92670b8
++ expected=1
++ myfield=systemd.u.SYSLOG_IDENTIFIER
++ wait_until_cmd_or_err test_count_expected test_count_err 300
++ let ii=300
++ local interval=1
++ '[' 300 -gt 0 ']'
++ test_count_expected
++ myfield=systemd.u.SYSLOG_IDENTIFIER
+++ query_es_from_es logging-es-ops-data-master-uuh5lj98-1-vbdzx .operations _count systemd.u.SYSLOG_IDENTIFIER a12e5b5d-afe3-46c5-b16d-315db92670b8
+++ get_count_from_json
+++ curl_es logging-es-ops-data-master-uuh5lj98-1-vbdzx '/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:a12e5b5d-afe3-46c5-b16d-315db92670b8' --connect-timeout 1
+++ local pod=logging-es-ops-data-master-uuh5lj98-1-vbdzx
+++ python -c 'import json, sys; print json.loads(sys.stdin.read()).get("count", 0)'
+++ local 'endpoint=/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:a12e5b5d-afe3-46c5-b16d-315db92670b8'
+++ shift
+++ shift
+++ args=("${@:-}")
+++ local args
+++ local secret_dir=/etc/elasticsearch/secret/
+++ oc exec logging-es-ops-data-master-uuh5lj98-1-vbdzx -- curl --silent --insecure --connect-timeout 1 --key /etc/elasticsearch/secret/admin-key --cert /etc/elasticsearch/secret/admin-cert 'https://localhost:9200/.operations*/_count?q=systemd.u.SYSLOG_IDENTIFIER:a12e5b5d-afe3-46c5-b16d-315db92670b8'
++ local nrecs=1
++ test 1 = 1
++ break
++ '[' 300 -le 0 ']'
++ return 0
good - wait_for_fluentd_to_catch_up: found 1 record project .operations for a12e5b5d-afe3-46c5-b16d-315db92670b8
++ echo good - wait_for_fluentd_to_catch_up: found 1 record project .operations for a12e5b5d-afe3-46c5-b16d-315db92670b8
++ '[' -n get_logmessage ']'
++ get_logmessage 869f7d82-5866-4867-a749-53e06dfe24d5
++ logmessage=869f7d82-5866-4867-a749-53e06dfe24d5
++ '[' -n get_logmessage2 ']'
++ get_logmessage2 a12e5b5d-afe3-46c5-b16d-315db92670b8
++ logmessage2=a12e5b5d-afe3-46c5-b16d-315db92670b8
+++ date +%s
++ local endtime=1496949290
+++ expr 1496949290 - 1496949278
+++ date -u --rfc-3339=ns
END wait_for_fluentd_to_catch_up took 12 seconds at 2017-06-08 19:14:50.995198420+00:00
++ echo END wait_for_fluentd_to_catch_up took 12 seconds at 2017-06-08 19:14:50.995198420+00:00
++ return 0
+++ get_running_pod kibana
+++ oc get pods -l component=kibana
+++ awk -v sel=kibana '$1 ~ sel && $3 == "Running" {print $1}'
++ kpod=logging-kibana-1-86619
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-86619 logging-es project.logging _search message 869f7d82-5866-4867-a749-53e06dfe24d5
++ python test-viaq-data-model.py test5 allow_empty
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer xQLLRNSoVy0CkrprwHB0cNYkyG-yXiqnF78837UFjXk' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es:9200/project.logging*/_search?q=message:869f7d82-5866-4867-a749-53e06dfe24d5'
++ :
++ '[' 0 = 0 ']'
++ curl_es_from_kibana logging-kibana-1-86619 logging-es-ops .operations _search message a12e5b5d-afe3-46c5-b16d-315db92670b8
++ python test-viaq-data-model.py test5 allow_empty
++ oc exec logging-kibana-1-86619 -c kibana -- curl --connect-timeout 1 -s -k --cert /etc/kibana/keys/cert --key /etc/kibana/keys/key -H 'X-Proxy-Remote-User: admin' -H 'Authorization: Bearer xQLLRNSoVy0CkrprwHB0cNYkyG-yXiqnF78837UFjXk' -H 'X-Forwarded-For: 127.0.0.1' 'https://logging-es-ops:9200/.operations*/_search?q=message:a12e5b5d-afe3-46c5-b16d-315db92670b8'
++ :
++ '[' 0 '!=' 0 ']'
++ return 0
++ cleanup
++ remove_test_volume
++ oc get template logging-fluentd-template -o json
++ python -c 'import json, sys; obj = json.loads(sys.stdin.read()); vm = obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"]; obj["objects"][0]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] = [xx for xx in vm if xx["name"] != "cdmtest"]; vs = obj["objects"][0]["spec"]["template"]["spec"]["volumes"]; obj["objects"][0]["spec"]["template"]["spec"]["volumes"] = [xx for xx in vs if xx["name"] != "cdmtest"]; print json.dumps(obj, indent=2)'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ remove_cdm_env
++ oc get template logging-fluentd-template -o yaml
++ sed '/- name: CDM_/,/value:/d'
++ oc replace -f -
template "logging-fluentd-template" replaced
++ rm -f /tmp/tmp.s1tQLSzDYA
++ restart_fluentd
++ oc delete daemonset logging-fluentd
daemonset "logging-fluentd" deleted
++ wait_for_pod_ACTION stop logging-fluentd-6mtlj
++ local ii=120
++ local incr=10
++ '[' stop = start ']'
++ curpod=logging-fluentd-6mtlj
++ '[' -z logging-fluentd-6mtlj -a -n '' ']'
++ '[' 120 -gt 0 ']'
++ '[' stop = stop ']'
++ oc describe pod/logging-fluentd-6mtlj
++ '[' stop = start ']'
++ break
++ '[' 120 -le 0 ']'
++ return 0
++ oc process logging-fluentd-template
++ oc create -f -
daemonset "logging-fluentd" created
++ wait_for_pod_ACTION start fluentd
++ local ii=120
++ local incr=10
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=
++ '[' 120 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z '' ']'
++ '[' -n '' ']'
++ '[' -n 1 ']'
pod for component=fluentd not running yet
++ echo pod for component=fluentd not running yet
++ sleep 10
+++ expr 120 - 10
++ ii=110
++ '[' start = start ']'
+++ get_running_pod fluentd
+++ oc get pods -l component=fluentd
+++ awk -v sel=fluentd '$1 ~ sel && $3 == "Running" {print $1}'
++ curpod=logging-fluentd-bl3pd
++ '[' 110 -gt 0 ']'
++ '[' start = stop ']'
++ '[' start = start ']'
++ '[' -z logging-fluentd-bl3pd ']'
++ break
++ '[' 110 -le 0 ']'
++ return 0
SKIPPING reinstall test for now
/data/src/github.com/openshift/origin-aggregated-logging/hack/lib/log/system.sh: line 31:  4609 Terminated              sar -A -o "${binary_logfile}" 1 86400 > /dev/null 2> "${stderr_logfile}"  (wd: /data/src/github.com/openshift/origin-aggregated-logging)
[INFO] [CLEANUP] Beginning cleanup routines...
[INFO] [CLEANUP] Dumping cluster events to /tmp/origin-aggregated-logging/artifacts/events.txt
[INFO] [CLEANUP] Dumping etcd contents to /tmp/origin-aggregated-logging/artifacts/etcd
[WARNING] No compiled `etcdhelper` binary was found. Attempting to build one using:
[WARNING]   $ hack/build-go.sh tools/etcdhelper
++ Building go targets for linux/amd64: tools/etcdhelper
/data/src/github.com/openshift/origin-aggregated-logging/../origin/hack/build-go.sh took 439 seconds
2017-06-08 15:22:50.347139 I | warning: ignoring ServerName for user-provided CA for backwards compatibility is deprecated
[INFO] [CLEANUP] Dumping container logs to /tmp/origin-aggregated-logging/logs/containers
[INFO] [CLEANUP] Truncating log files over 200M
[INFO] [CLEANUP] Stopping docker containers
[INFO] [CLEANUP] Removing docker containers
Error response from daemon: You cannot remove a running container 34eb5fdc1c34f93eed24fc284897535a339d2f4f9f865679e566c9b99ed84b51. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 79e940e8453f8a1857a45d668fd002120d5a61c657dd864b7b31a13aa5eb5f78. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container a138b03c59399f6e5b35dd1240f03a6a01534695a55f9369c0d98160da5c2d17. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 3e3d893638bf9c28c5303e6d744c6078fee44166bab6e9494446e04c3ba3fb68. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 973870e1d564340ca5078e763b37bb11d938449e3d35bc70db8886692f1e411b. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 0423e4d77fb32c43b5cbee702f3e2f68e150769d2e9d85329fcebbb19dccc008. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 3a72296aeb135f269e3391845f769a5798cbec547bf61d5451d8b8edfa38da16. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 5d0b676955f21369a89f2c14ab23e5083dafe0b64980627fa0f3a79b8ed3786d. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 78c2fa63f5ad210a32953606c118735ebbccec564e8b603ae7beb14045f7c958. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container c59aa0ce1822c05579e4e334eb5f4101de0655a61b9a9e44b840cc1947292d23. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container daf8cb90e4bf4059d85e8f9015b787168d5b7e3add3f62013a373393b78a4606. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 5935045927e5851e7e81afb5308b6c469fe86345764c63208b2587aec109dd3e. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container cec39ba0ba218d63f05c1850fa07704ea472247fe9e90361fd8449dfa649836e. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container b70d4e6b3965c1407abb3f26bf75455f71f20c2dc80301102db8733a95f05355. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container 58595d2737a7be5e74a822ffe0275d7d1a5438085e4d48268e235edd77738340. Stop the container before attempting removal or use -f
Error response from daemon: You cannot remove a running container e529f82acbb75c278817e2b6ec3fe53ba690a2718438a56cc094a9595af19860. Stop the container before attempting removal or use -f
[INFO] [CLEANUP] Killing child processes
[INFO] [CLEANUP] Pruning etcd data directory
[INFO] /data/src/github.com/openshift/origin-aggregated-logging/logging.sh exited with code 0 after 01h 05m 50s

real	65m49.906s
user	5m27.602s
sys	1m0.270s
Finished GIT_URL=https://github.com/openshift/origin-aggregated-logging GIT_BRANCH=master O_A_L_DIR=/data/src/github.com/openshift/origin-aggregated-logging OS_ROOT=/data/src/github.com/openshift/origin ENABLE_OPS_CLUSTER=true USE_LOCAL_SOURCE=true TEST_PERF=false VERBOSE=1 OS_ANSIBLE_REPO=https://github.com/openshift/openshift-ansible OS_ANSIBLE_BRANCH=master ./logging.sh
***************************************************
==> openshiftdev: Downloading logs
==> openshiftdev: Downloading artifacts from '/var/log/yum.log' to '/var/lib/jenkins/jobs/test-origin-aggregated-logging/workspace@2/origin/artifacts/yum.log'
==> openshiftdev: Downloading artifacts from '/var/log/secure' to '/var/lib/jenkins/jobs/test-origin-aggregated-logging/workspace@2/origin/artifacts/secure'
==> openshiftdev: Downloading artifacts from '/var/log/audit/audit.log' to '/var/lib/jenkins/jobs/test-origin-aggregated-logging/workspace@2/origin/artifacts/audit.log'
==> openshiftdev: Downloading artifacts from '/tmp/origin-aggregated-logging/' to '/var/lib/jenkins/jobs/test-origin-aggregated-logging/workspace@2/origin/artifacts'
+ false
+ false
+ test_pull_requests --mark_test_success --repo origin-aggregated-logging --config /var/lib/jenkins/.test_pull_requests_logging.json
Rate limit remaining: 2303
  Marking SUCCESS for pull request #--repo in repo ''
/usr/share/ruby/net/http/response.rb:119:in `error!': 404 "Not Found" (Net::HTTPServerException)
	from /bin/test_pull_requests:543:in `block in get_comments'
	from /bin/test_pull_requests:535:in `each'
	from /bin/test_pull_requests:535:in `get_comments'
	from /bin/test_pull_requests:1074:in `get_comment_matching_regex'
	from /bin/test_pull_requests:1090:in `get_comment_with_prefix'
	from /bin/test_pull_requests:779:in `mark_test_success'
	from /bin/test_pull_requests:2361:in `<main>'
+ true
[description-setter] Could not determine description.
[PostBuildScript] - Execution post build scripts.
[workspace@2] $ /bin/sh -xe /tmp/hudson6182997038383821608.sh
+ INSTANCE_NAME=origin_logging-rhel7-1635
+ pushd origin
~/jobs/test-origin-aggregated-logging/workspace@2/origin ~/jobs/test-origin-aggregated-logging/workspace@2
+ rc=0
+ '[' -f .vagrant-openshift.json ']'
++ /usr/bin/vagrant ssh -c 'sudo ausearch -m avc'
+ ausearchresult='<no matches>'
+ rc=1
+ '[' '<no matches>' = '<no matches>' ']'
+ rc=0
+ /usr/bin/vagrant destroy -f
==> openshiftdev: Terminating the instance...
==> openshiftdev: Running cleanup tasks for 'shell' provisioner...
+ popd
~/jobs/test-origin-aggregated-logging/workspace@2
+ exit 0
Finished: SUCCESS