Single or Multiple

Question

Single or Multiple

Closed this issue a year ago · 6 comments

Hi,

Thanks for the work on this module looks great, I do have a couple of questions.

I am trying to write a data check and I need to be able to distinguish between the following different type of json record.

Case 1

{"name": "Paul","Age": 25,"Location": "USA"}

Case 2

[
{"name": "Paul","Age": 25,"Location": "USA"} ,
{"name": "Fred","Age": 27,"Location": "UK"} ,
{"name": "John","Age": 20,"Location": "UAE"} 
]

Case 3

{"name": "Paul","Age": 25,"Location": "USA"} 
{"name": "Fred","Age": 27,"Location": "UK"} 
{"name": "John","Age": 20,"Location": "UAE"}

Case 1 is the standard simple approach but if the data comes through in either Case 2 or 3 I need a way to identify this and then be able to iterate over this using a range so that I can not only use Get but also use the single string line for other data.

Any ideas on the best way to do this?

Mark

Answer 1 · 2023-07-28T10:50:32.000Z

With a bit of playing around I got nearly the output needed using the following code:

https://go.dev/play/p/zQoFPc0l8Vi

package main

import (
	"bufio"
	"fmt"
	"strings"

	"github.com/ohler55/ojg"
	"github.com/ohler55/ojg/jp"
	"github.com/ohler55/ojg/oj"
)

var (
	event1 = `{"name": "Paul","Age": 25,"Location": "USA"}`
	event2 = `[
	{"name": "Paul","Age": 25,"Location": "USA"},
	{"name": "Fred","Age": 27,"Location": "UK"},
	{"name": "John","Age": 20,"Location": "UAE"}
	]`
	event3 = `{"name": "Paul","Age": 25,"Location": "USA"}
	{"name": "Fred","Age": 27,"Location": "UK"}
	{"name": "John","Age": 20,"Location": "UAE"}`
)

func getJsonStrings(data string) []string {
	var result []string
	obj, err := oj.ParseString(data)
	if err != nil {
		fmt.Println("going to try and parse as json lines")
		obj = convertJsonLines(data)
	}
	switch obj.(type) {
	case []interface{}:
		for _, v := range obj.([]interface{}) {
			parse := localParse(v)
			var b strings.Builder
			if err := oj.Write(&b, parse, &ojg.Options{Sort: true}); err != nil {
				panic(err)
			}
			result = append(result, b.String())
		}
	case map[string]interface{}:
		parse := localParse(obj)
		var b strings.Builder
		if err := oj.Write(&b, parse, &ojg.Options{Sort: true}); err != nil {
			panic(err)
		}
		result = append(result, b.String())
	default:
		panic("unknown type")
	}
	return result
}

func localParse(obj any) any {
	x, err := jp.ParseString("$.name")
	if err != nil {
		fmt.Println(fmt.Sprintf("error parsing json: %s", err))
	}
	result := x.Get(obj)
	return result
}

func convertJsonLines(data string) any {
	var (
		payload []string
	)
	scanner := bufio.NewScanner(strings.NewReader(data))
	for scanner.Scan() {
		payload = append(payload, scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		fmt.Printf("error occurred: %v\n", err)
	}
	result := strings.Join(payload, ",")
	obj, err := oj.ParseString("[" + result + "]")
	if err != nil {
		fmt.Println(fmt.Sprintf("error parsing jsonlines: %s", err))
	}
	return obj
}

func main() {
	var tests = []struct {
		name  string
		event string
		topic []string
	}{
		{"single json event", event1, []string{"Paul"}},
		{"map of json events", event2, []string{"Paul", "Fred", "John"}},
		{"multiple json line events", event3, []string{"Paul", "Fred", "John"}},
	}
	{
		for _, tt := range tests {
			fmt.Println(fmt.Sprintf("running test: %s", tt.name))
			result := getJsonStrings(tt.event)
			fmt.Println(fmt.Sprintf("result: %s", result))

			if result[0] != tt.topic[0] {
				fmt.Println(fmt.Sprintf("got %s, want %s", result, tt.topic))
			} else {
				fmt.Println("success correct result")
			}
			fmt.Println("")
		}
	}
}

But it looks like i am having an issue using x.Get(obj) which returns any when cast to string it get [data] rather than data?

How do i resolve that?

Is there a better way to rewrite this?

Answer 2 · 2023-07-28T11:57:51.000Z

Well with a little more playing and a bit of trim I think the following is one way of achiving my result not sure it if the best though.

package main

import (
	"bufio"
	"fmt"
	"strings"

	"github.com/ohler55/ojg"
	"github.com/ohler55/ojg/jp"
	"github.com/ohler55/ojg/oj"
)

var (
	event1 = `{"name": "Paul","Age": 25,"Location": { "country": "USA", "city": "New York" }}`
	event2 = `[
	{"name": "Paul","Age": 25,"Location": { "country": "USA", "city": "New York" }},
	{"name": "Fred","Age": 27,"Location": { "country": "UK", "city": "London" }},
	{"name": "John","Age": 20,"Location": { "country": "UAE", "city": "Dubai" }}
	]`
	event3 = `{"name": "Paul","Age": 25,"Location": { "country": "USA", "city": "New York" }}
	{"name": "Fred","Age": 27,"Location": { "country": "UK", "city": "London" }}
	{"name": "John","Age": 20,"Location": { "country": "UAE", "city": "Dubai" }}`
)


func jsonConversion(data string) interface{} {
	obj, err := oj.ParseString(data)
	if err != nil {
		fmt.Println("Attempting to parse as json lines...")
		return convertJsonLines(data)
	}
	return obj
}

func writeJson(parsedData interface{}) string {
	var writer strings.Builder
	if err := oj.Write(&writer, parsedData, &ojg.Options{Sort: true}); err != nil {
		panic(err)
	}
	fmt.Println(writer.String())
	return writer.String()
}

func getJSONStrings(data string, jsonString string) []string {
	var result []string
	jsonObj := jsonConversion(data)

	switch parsedObj := jsonObj.(type) {
	case []interface{}:
		result = appendItems(result, jsonString, parsedObj...)
	case map[string]interface{}:
		result = appendItem(result, jsonString, parsedObj)
	default:
		panic("Unknown type")
	}
	return result
}

func appendItems(slice []string, jsonString string, items ...interface{}) []string {
	for _, item := range items {
		slice = appendItem(slice, jsonString, item)
	}
	return slice
}

func appendItem(slice []string, jsonString string, item interface{}) []string {
	jsonStr := writeJson(localParse(item, jsonString))
	trimmedStr := trimEdges(jsonStr)
	return append(slice, trimmedStr)
}

func trimEdges(jsonString string) string {
	return strings.Trim(
		strings.Trim(jsonString, "\"]"),
		"[\"",
	)
}

func localParse(obj interface{}, jsonPath string) interface{} {
	path, _ := jp.ParseString(jsonPath)
	return path.Get(obj)
}

func convertJsonLines(data string) interface{} {
	var payload []string
	scanner := bufio.NewScanner(strings.NewReader(data))
	for scanner.Scan() {
		payload = append(payload, scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		fmt.Printf("Error occurred: %v\n", err)
	}

	combinedLines := fmt.Sprintf("[%s]", strings.Join(payload, ","))
	obj, err := oj.ParseString(combinedLines)
	if err != nil {
		fmt.Printf("Error parsing jsonlines: %s\n", err)
	}
	return obj
}

func main() {
	var tests = []struct {
		name  string
		event string
		jsonPath string
		topic []string
	}{
		{"single json event - name", event1, "$.name", []string{"Paul"}},
		{"map of json events - name", event2, "$.name", []string{"Paul", "Fred", "John"}},
		{"multiple json line events - name", event3, "$.name", []string{"Paul", "Fred", "John"}},
		{"single json event - location", event1, "$.Location", []string{"{\"city\":\"New York\",\"country\":\"USA\"}"}},
		{"map of json events - location", event2, "$.Location", []string{"{\"city\":\"New York\",\"country\":\"USA\"}", "{\"city\":\"London\",\"country\":\"UK\"}", "{\"city\":\"Dubai\",\"country\":\"UAE\"}"}},
		{"multiple json line events - location", event3, "$.Location", []string{"{\"city\":\"New York\",\"country\":\"USA\"}", "{\"city\":\"London\",\"country\":\"UK\"}", "{\"city\":\"Dubai\",\"country\":\"UAE\"}"}},
	}
	{
		for _, tt := range tests {
			fmt.Println(fmt.Sprintf("running test: %s", tt.name))
			result := getJSONStrings(tt.event, tt.jsonPath)
			fmt.Println(fmt.Sprintf("result: %s", result))

			if result[0] != tt.topic[0] {
				fmt.Println(fmt.Sprintf("got %s, want %s", result, tt.topic))
			} else {
				fmt.Println("success correct result")
			}
			fmt.Println("")
		}
	}
}

https://go.dev/play/p/ZZvRD9y8lXZ

Answer 3 · 2023-09-08T23:31:36.000Z

My apologies for taking so very long to reply. Work and then a vacation were a little too distracting. If you are still stuck I'd be glad to explain a different and simpler approach.

Answer 4 · 2023-09-09T05:47:25.000Z

Hi,

No worries, I think I have it working now just but more than happy to learn a different approach especially if it is simpler.

Thanks

Mark

Answer 5 · 2023-09-09T12:02:41.000Z

This is another approach:

package main

import (
	"fmt"

	"github.com/ohler55/ojg/jp"
	"github.com/ohler55/ojg/oj"
)

var (
	event1 = `{"name": "Paul","Age": 25,"Location": "USA"}`
	event2 = `[
	{"name": "Paul","Age": 25,"Location": "USA"},
	{"name": "Fred","Age": 27,"Location": "UK"},
	{"name": "John","Age": 20,"Location": "UAE"}
	]`
	event3 = `{"name": "Paul","Age": 25,"Location": "USA"}
	{"name": "Fred","Age": 27,"Location": "UK"}
	{"name": "John","Age": 20,"Location": "UAE"}`
)

func main() {
	var tests = []struct {
		name  string
		event string
		topic []string
	}{
		{"single json event", event1, []string{"Paul"}},
		{"map of json events", event2, []string{"Paul", "Fred", "John"}},
		{"multiple json line events", event3, []string{"Paul", "Fred", "John"}},
	}
top:
	for _, tt := range tests {
		fmt.Printf("running test: %s\n", tt.name)
		result := getJsonStrings(tt.event)
		fmt.Printf("result: %s\n", result)

		if len(tt.topic) != len(result) {
			fmt.Printf("got %s, want %s\n", result, tt.topic)
			continue
		}
		for i, name := range tt.topic {
			if name != result[i] {
				fmt.Printf("got %s, want %s\n", result, tt.topic)
				continue top
			}
		}
		fmt.Println("success correct result")
	}
}

func getJsonStrings(data string) (result []string) {
	_ = oj.MustParseString(data, func(v any) bool {
		if list, ok := v.([]any); ok {
			for _, v2 := range list {
				if name, _ := jp.C("name").First(v2).(string); 0 < len(name) {
					result = append(result, name)
				}
			}
		} else {
			if name, _ := jp.C("name").First(v).(string); 0 < len(name) {
				result = append(result, name)
			}
		}
		return false
	})
	return
}

Answer 6 · 2023-09-13T23:57:21.000Z

Did the example help? If so can this be closed?