From 54bb5a070defa7255b105670f57c4d45f6c17df9 Mon Sep 17 00:00:00 2001 From: Junwei Zhao Date: Sun, 1 Mar 2020 22:13:16 +1100 Subject: [PATCH] Add functionality to build index for organisation, user and tickets --- main.go | 6 +- search/app.go | 11 +++- search/index.go | 158 ++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 152 insertions(+), 23 deletions(-) diff --git a/main.go b/main.go index 4d69bce..cb5204d 100644 --- a/main.go +++ b/main.go @@ -1,8 +1,6 @@ package main import ( - "fmt" - "github.com/wahyd4/zendesk/search" ) @@ -12,8 +10,8 @@ func main() { if err := app.Parse(); err != nil { panic("cannot load data: " + err.Error()) } - if err := app.BuildOrganisationIndex(); err != nil { + if err := app.BuildIndexes(); err != nil { panic("failed to build indexes: " + err.Error()) } - fmt.Println("---------------") + } diff --git a/search/app.go b/search/app.go index 69aa4d7..36f4ea8 100644 --- a/search/app.go +++ b/search/app.go @@ -20,8 +20,8 @@ type APP struct { tickets map[string]*model.Ticket organisationIndex map[string]map[string][]*model.Organisation - // userIndex map[string]map[string][]*model.User - // ticketIndex map[string]map[string][]*model.Ticket + userIndex map[string]map[string][]*model.User + ticketIndex map[string]map[string][]*model.Ticket } func InitAPP(organisationsFile, usersFile, ticketsFile string) *APP { @@ -42,6 +42,8 @@ func InitAPP(organisationsFile, usersFile, ticketsFile string) *APP { return &APP{ jsonContents: jsonContents, organisationIndex: make(map[string]map[string][]*model.Organisation), + userIndex: make(map[string]map[string][]*model.User), + ticketIndex: make(map[string]map[string][]*model.Ticket), } } @@ -54,3 +56,8 @@ func (app *APP) FindOrganisation(id string) *model.Organisation { func (app *APP) FindUser(id string) *model.User { return app.users[id] } + +// FindTicket find a ticket by ticket ID +func (app *APP) FindTicket(id string) *model.Ticket { + return app.tickets[id] +} diff --git a/search/index.go b/search/index.go index 919f8c4..3ce40b3 100644 --- a/search/index.go +++ b/search/index.go @@ -11,13 +11,31 @@ const ( idKey = "_id" ) +type IndexBuilderFunc func() error + var ( arrayKeys = []string{"domain_names", "tags"} - booleanKeys = []string{"shared_tickets"} - integerKeys = []string{"_id"} + booleanKeys = []string{"shared_tickets", "active", "verified", "shared", "suspended", "has_incidents"} + integerKeys = []string{"_id", "organization_id", "submitter_id", "assignee_id"} ) -func (app *APP) BuildOrganisationIndex() error { +// BuildIndexes build indexes for searching +func (app *APP) BuildIndexes() error { + indexBuilders := map[string]IndexBuilderFunc{ + OrganisationsKey: app.buildOrganisationIndex, + UsersKey: app.buildUserIndex, + TicketsKey: app.buildTicketIndex, + } + + for indexType, indexBuilderFunc := range indexBuilders { + if err := indexBuilderFunc(); err != nil { + return fmt.Errorf("failed to build index for %s with error: %w", indexType, err) + } + } + return nil +} + +func (app *APP) buildOrganisationIndex() error { bytes := app.jsonContents[OrganisationsKey] var organisations []map[string]interface{} @@ -31,6 +49,8 @@ func (app *APP) BuildOrganisationIndex() error { return nil } searchIndex := app.organisationIndex + + // get first entity from json array as the template organisationsTemplate := organisations[0] for fieldKey := range organisationsTemplate { @@ -41,21 +61,9 @@ func (app *APP) BuildOrganisationIndex() error { if searchIndex[fieldKey] == nil { searchIndex[fieldKey] = make(map[string][]*model.Organisation) } - fieldIndex := searchIndex[fieldKey] - fieldValues := make([]string, 0) - - // process value based on different types - if OneOfTheKeys(integerKeys, fieldKey) { - fieldValues = append(fieldValues, stringifyID(int(organisation[fieldKey].(float64)))) - } else if OneOfTheKeys(booleanKeys, fieldKey) { - fieldValues = append(fieldValues, fmt.Sprintf("%t", organisation[fieldKey].(bool))) - } else if OneOfTheKeys(arrayKeys, fieldKey) { - fieldValues = toStringSlice(organisation[fieldKey].([]interface{})) - } else { - fieldValues = append(fieldValues, organisation[fieldKey].(string)) - } + fieldValues := extractFieldValues(organisation, fieldKey) // build fieldValue based indeX for _, fieldValue := range fieldValues { @@ -72,7 +80,123 @@ func (app *APP) BuildOrganisationIndex() error { return nil } -func OneOfTheKeys(keys []string, target string) bool { +func (app *APP) buildUserIndex() error { + bytes := app.jsonContents[UsersKey] + + var users []map[string]interface{} + + err := json.Unmarshal(bytes, &users) + if err != nil { + return fmt.Errorf("failed to unmarshal json to user array: %w", err) + } + + if len(users) == 0 { + return nil + } + searchIndex := app.userIndex + + // get first entity from json array as the template + entityTemplate := users[0] + + for fieldKey := range entityTemplate { + + for _, user := range users { + userID := stringifyID(int(user[idKey].(float64))) + + if searchIndex[fieldKey] == nil { + searchIndex[fieldKey] = make(map[string][]*model.User) + } + fieldIndex := searchIndex[fieldKey] + + fieldValues := extractFieldValues(user, fieldKey) + + // build fieldValue based indeX + for _, fieldValue := range fieldValues { + if fieldIndex[fieldValue] == nil { + sameValueList := make([]*model.User, 0) + fieldIndex[fieldValue] = sameValueList + } + + fieldIndex[fieldValue] = append(fieldIndex[fieldValue], app.FindUser(userID)) + } + + } + } + return nil +} + +func (app *APP) buildTicketIndex() error { + bytes := app.jsonContents[TicketsKey] + + var tickets []map[string]interface{} + + err := json.Unmarshal(bytes, &tickets) + if err != nil { + return fmt.Errorf("failed to unmarshal json to ticket array: %w", err) + } + + if len(tickets) == 0 { + return nil + } + searchIndex := app.ticketIndex + + // get first entity from json array as the template + entityTemplate := tickets[0] + + for fieldKey := range entityTemplate { + + for _, ticket := range tickets { + ticketID := ticket[idKey].(string) + + if searchIndex[fieldKey] == nil { + searchIndex[fieldKey] = make(map[string][]*model.Ticket) + } + fieldIndex := searchIndex[fieldKey] + var fieldValues []string + + // due to the ticket id is string + if fieldKey == idKey { + fieldValues = []string{ticketID} + } else { + fieldValues = extractFieldValues(ticket, fieldKey) + } + + // build fieldValue based indeX + for _, fieldValue := range fieldValues { + if fieldIndex[fieldValue] == nil { + sameValueList := make([]*model.Ticket, 0) + fieldIndex[fieldValue] = sameValueList + } + + fieldIndex[fieldValue] = append(fieldIndex[fieldValue], app.FindTicket(ticketID)) + } + + } + } + return nil +} + +func extractFieldValues(entity map[string]interface{}, fieldKey string) []string { + fieldValues := make([]string, 0) + + if entity[fieldKey] == nil { + return fieldValues + } + // process value based on different types + if oneOfTheKeys(integerKeys, fieldKey) { + fieldValues = append(fieldValues, stringifyID(int(entity[fieldKey].(float64)))) + } else if oneOfTheKeys(booleanKeys, fieldKey) { + fieldValues = append(fieldValues, fmt.Sprintf("%t", entity[fieldKey].(bool))) + } else if oneOfTheKeys(arrayKeys, fieldKey) { + fieldValues = toStringSlice(entity[fieldKey].([]interface{})) + } else { + fieldValues = append(fieldValues, entity[fieldKey].(string)) + } + + return fieldValues +} + +func oneOfTheKeys(keys []string, target string) bool { for _, key := range keys { if key == target { return true