parse_yaml script is not working or identifying if the yamlfile has any issues
Opened this issue · 3 comments
vivuu1989 commented
We are using Linux machine of Red Hat Enterprise Linux Server release 7.9 (Maipo) and Ubuntu 18. We have noticed some strange issue when using the parse-yaml.sh script is that, when we preform source parse-yaml.sh , and after that when executing some parse command such as parse-yaml test.yaml, the command will be hung for long time without giving any output.
By enabling the debug, I could understand that its getting stuck in + cat dev-policy.yaml, i
-bash-4.2$ parse_yaml dev-policy.yaml policy
+ parse_yaml dev-policy.yaml policy
+ unset i
+ unset fs
+ local prefix=policy
+ local separator=_
+ local indexfix=-1
+ grep --color=auto -q 'GNU Awk'
+ awk --version
+ indexfix=-1
++ echo @
++ tr @ '\034'
+ local 's=[[:space:]]*' 'sm=[ \t]*' 'w=[a-zA-Z0-9_.]*' $'fs=\034' 'i= '
+ awk $'-F\034' '{multi=0;
if(match($0,/[ \t]*\|[ \t]*$/)){multi=1; sub(/[ \t]*\|[ \t]*$/,"");}
if(match($0,/[ \t]*>[ \t]*$/)){multi=2; sub(/[ \t]*>[ \t]*$/,"");}
while(multi>0){
str=$0; gsub(/^[ \t]*/,"", str);
indent=index($0,str);
indentstr=substr($0, 0, indent+-1) " ";
obuf=$0;
getline;
while(index($0,indentstr)){
obuf=obuf substr($0, length(indentstr)+1);
if (multi==1){obuf=obuf "\\n";}
if (multi==2){
if(match($0,/^[ \t]*$/))
obuf=obuf "\\n";
else obuf=obuf " ";
}
getline;
}
sub(/[ \t]*$/,"",obuf);
print obuf;
multi=0;
if(match($0,/[ \t]*\|[ \t]*$/)){multi=1; sub(/[ \t]*\|[ \t]*$/,"");}
if(match($0,/[ \t]*>[ \t]*$/)){multi=2; sub(/[ \t]*>[ \t]*$/,"");}
}
print}'
+ sed -ne 's|,[[:space:]]*\]|]|g' -e :1 -e 's|^\([[:space:]]*\)\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*\(&[a-zA-Z0-9_.]*\)[[:space:]]*\[[[:space:]]*\(.*\)[[:space:]]*,[[:space:]]*\(.*\)[[:space:]]*\]|\1\2: \3[\4]\n\1 - \5|;t 1' -e 's|^\([[:space:]]*\)\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*\(&[a-zA-Z0-9_.]*\)[[:space:]]*\[[[:space:]]*\(.*\)[[:space:]]*\]|\1\2: \3\n\1 - \4|;' -e :2 -e 's|^\([[:space:]]*\)\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*\[[[:space:]]*\(.*\)[[:space:]]*,[[:space:]]*\(.*\)[[:space:]]*\]|\1\2: [\3]\n\1 - \4|;t 2' -e 's|^\([[:space:]]*\)\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*\[[[:space:]]*\(.*\)[[:space:]]*\]|\1\2:\n\1 - \3|;' -e :3 -e 's|^\([[:space:]]*\)-[[:space:]]*\[[[:space:]]*\(.*\)[[:space:]]*,[[:space:]]*\(.*\)[[:space:]]*\]|\1- [\2]\n\1 - \3|;t 3' -e 's|^\([[:space:]]*\)-[[:space:]]*\[[[:space:]]*\(.*\)[[:space:]]*\]|\1-\n\1 - \2|;p'
+ sed -ne 's|^\([[:space:]]*\):|\1|' -e 's|^\([[:space:]]*\)\(---\)\([[:space:]]*\)||' -e 's|^\([[:space:]]*\)\(\.\.\.\)\([[:space:]]*\)||' -e 's|^\([[:space:]]*\)-[[:space:]]*["'\'']\(.*\)["'\''][[:space:]]*$|\1\2|p;t' -e 's|^\([[:space:]]*\)\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*["'\'']\(.*\)["'\''][[:space:]]*$|\1\2\3|p;t' -e 's|^\([[:space:]]*\)-[[:space:]]*\(.*\)[[:space:]]*$|\1\2|' -e 's|^\([[:space:]]*\)\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*["'\'']\?\(.*\)[[:space:]]*$|\1\2\3|' -e 's|^\([[:space:]]*\)["'\'']\?\([^&][^]\+\)["'\''][[:space:]]*$|\1\2|' -e 's|^\([[:space:]]*\)["'\'']\?\([^&][^]\+\)[[:space:]]*$|\1\2|' -e 's|^\([[:space:]]*\)\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*["'\'']\(.*\)[[:space:]]*$|\1\2\3|' -e 's|^\([[:space:]]*\)["'\'']\([^&][^]*\)["'\''][[:space:]]*$|\1\2|' -e 's|^\([[:space:]]*\)["'\'']\([^&][^]*\)[[:space:]]*$|\1\2|' -e 's|^\([[:space:]]*\)\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*\(.*\)[[:space:]]*$|\1\2\3|' -e 's|^\([[:space:]]*\)\([^&][^]*\)["'\''][[:space:]]*$|\1\2|' -e 's|^\([[:space:]]*\)\([^&][^]*\)[[:space:]]*$|\1\2|' -e 's|[[:space:]]*$||p'
+ awk $'-F\034' '{
gsub(/\t/," ",$1);
if(NF>3){if(value!=""){value = value " ";}value = value $4;}
else {
if(match($1,/^&/)){anchor[substr($1,2)]=full_vn;getline};
indent = length($1)/length(" ");
vname[indent] = $2;
value= $3;
for (i in vname) {if (i > indent) {delete vname[i]; idx[i]=0}}
if(length($2)== 0){ vname[indent]= ++idx[indent] };
vn=""; for (i=0; i<indent; i++) { vn=(vn)(vname[i])("_")}
vn="policy" vn;
full_vn=vn vname[indent];
if(vn=="policy")vn="policy_";
if(vn=="_")vn="__";
}
gsub(/\./,"_",full_vn);
gsub(/\\"/,"\"",value);
gsub(/'\''/,"'\''\"'\''\"'\''",value);
assignment[full_vn]=value;
if(!match(assignment[vn], full_vn))assignment[vn]=assignment[vn] " " full_vn;
if(match(value,/^\*/)){
ref=anchor[substr(value,2)];
if(length(ref)==0){
printf("%s='\''%s'\''\n", full_vn, value);
} else {
for(val in assignment){
if((length(ref)>0)&&index(val, ref)==1){
tmpval=assignment[val];
sub(ref,full_vn,val);
if(match(val,"_$")){
gsub(ref,full_vn,tmpval);
} else if (length(tmpval) > 0) {
printf("%s='\''%s'\''\n", val, tmpval);
}
assignment[val]=tmpval;
}
}
}
} else if (length(value) > 0) {
printf("%s='\''%s'\''\n", full_vn, value);
}
}END{
for(val in assignment){
if(match(val,"_$"))
printf("%s='\''%s'\''\n", val, assignment[val]);
}
}'
+ sed -ne 's|,[[:space:]]*}|}|g' -e :1 -e 's|^\([[:space:]]*\)-[[:space:]]*{[[:space:]]*\(.*\)[[:space:]]*,[[:space:]]*\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*\(.*\)[[:space:]]*}|\1- {\2}\n\1 \3: \4|;t 1' -e 's|^\([[:space:]]*\)-[[:space:]]*{[[:space:]]*\(.*\)[[:space:]]*}|\1-\n\1 \2|;' -e :2 -e 's|^\([[:space:]]*\)\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*\(&[a-zA-Z0-9_.]*\)[[:space:]]*{[[:space:]]*\(.*\)[[:space:]]*,[[:space:]]*\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*\(.*\)[[:space:]]*}|\1\2: \3 {\4}\n\1 \5: \6|;t 2' -e 's|^\([[:space:]]*\)\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*\(&[a-zA-Z0-9_.]*\)[[:space:]]*{[[:space:]]*\(.*\)[[:space:]]*}|\1\2: \3\n\1 \4|;' -e :3 -e 's|^\([[:space:]]*\)\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*{[[:space:]]*\(.*\)[[:space:]]*,[[:space:]]*\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*\(.*\)[[:space:]]*}|\1\2: {\3}\n\1 \4: \5|;t 3' -e 's|^\([[:space:]]*\)\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*{[[:space:]]*\(.*\)[[:space:]]*}|\1\2:\n\1 \3|;p'
+ sed -e 's|^\([[:space:]]*\)?|\1-|' -ne 's|^\([[:space:]]*\)-[[:space:]]*\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*\(.*\)|\1-\n\1 \2: \3|' -ne 's|^[[:space:]]*#.*||;s|[[:space:]]*#[^"'\'']*$||;s|^\([^"'\''#]*\)#.*|\1|;t 1' -ne t -ne :1 -ne 's|^[[:space:]]*$||;t 2' -ne p -ne :2 -ne d
+ sed -e 's|^\([[:space:]]*\)\([a-zA-Z0-9_.]*\)[[:space:]]*:[[:space:]]*\(&[a-zA-Z0-9_.]*\)\(.*\)|\1\2:\4\n\3|' -e 's|^\([[:space:]]*\)-[[:space:]]*\(&[a-zA-Z0-9_.]*\)\(.*\)|\1- \3\n\2|'
+ cat dev-policy.yaml
Here the dev-policy.yaml file is as below.
---
- apiname: myapi
scope: api
inboundsession: >
<base />
<choose>
<when condition="@(context.Request.Url.Path.Contains("xxxxxxx))">
<rate-limit-by-key calls="xxx" renewal-period="xx" counter-key="@(Regex.Match(context.Request.Headers.GetValueOrDefault("X-xxxxxxx-For",""), @"^[.0-9]*")?.Value)" />
</when>
<otherwise>
<check-header name="X-AssignedxxxxID" failed-check-httpcode="400" failed-check-error-message="Bad request" ignore-case="true" />
<rate-limit-by-key calls="xxxx" renewal-period="xx" counter-key="@(Regex.Match(context.Request.Headers.GetValueOrDefault("X-xxxxx-For",""), @"^[.0-9]*")?.Value)" />
<rate-limit-by-key calls="xxx" renewal-period="xx" counter-key="@(context.Request.Headers.GetValueOrDefault("X-AssignedxxxxxxID",""))" />
</otherwise>
</choose>
outboundsession: |
<base />
backendsession: |
<base />
onerrorsession: >
<base />
<choose>
<when condition="@(context.LastError.Reason == "RateLimitExceeded")">
<emit-metric name="policy_error_rate_limit_exceeded" value="1">
<dimension name="ClientIP" value="@(context.Request.Headers.GetValueOrDefault("xxxxxx","empty-ip"))" />
<dimension name="AssignedDeviceID" value="@(context.Request.Headers.GetValueOrDefault("X-AssignedxxxxxID","empty-xxxxx-id"))" />
<dimension name="xxxx ID" />
</emit-metric>
</when>
</choose>
- operationname: myapi
operationapi: myapi
scope: operation
inboundsession: |
<base />
outboundsession: |
<base />
backendsession: |
<base />
but when i create a fresh dev-policy.yaml file from scratch within the server itself, its parsing properly. Not sure how to fix this issue. Is this because of invalid input yaml file? if yes, how we can make correct validation before processing it ?
vivuu1989 commented
@mrbaseman , can u make look on this please
mrbaseman commented
I have seen this behavior sometimes, but couldn't find the reason.
Try to append another empty line or a comment at the end of the file.
My suspicion is that it's looking for a line to terminate a specific indentation. Usually, that's no problem, but maybe in combination with multiline input, something isn't working quite well.
vivuu1989 commented
Can we have a feature or way to fail the script parsing if the input file parsing didt work or yaml file input has wrong syntax.