Skip to content

Commit

Permalink
add read with selected fields tests
Browse files Browse the repository at this point in the history
  • Loading branch information
Cobalt0s committed Apr 8, 2024
1 parent 90f5934 commit 1381b97
Show file tree
Hide file tree
Showing 8 changed files with 180 additions and 26 deletions.
2 changes: 1 addition & 1 deletion common/interpreter/error.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
)

var (
ErrUnmarshal = errors.New("unmarshal failed")
ErrUnmarshal = errors.New("unmarshal failed")
MissingContentType = errors.New("mime.ParseMediaType failed")
)

Expand Down
7 changes: 6 additions & 1 deletion common/parse.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,12 @@ func (jsonManager) ArrSize(node *ajson.Node, key string) (int64, error) {
return int64(innerNode.Size()), nil
}

func (jsonManager) GetString(node *ajson.Node, key string) (string, error) {
func (jsonManager) GetString(node *ajson.Node, key string, optional bool) (string, error) {
if optional && !node.HasKey(key) {
// null value in payload is allowed
return "", nil
}

innerNode, err := node.GetKey(key)
if err != nil {
return "", err
Expand Down
6 changes: 3 additions & 3 deletions common/reqrepeater/standard.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,14 @@ func (r *UniformRetry) Completed() bool {
return false
}

// NullStrategy it doesn't even try
type NullStrategy struct {}
// NullStrategy it doesn't even try.
type NullStrategy struct{}

func (NullStrategy) Start() Retry {
return &NullRetry{}
}

type NullRetry struct {}
type NullRetry struct{}

func (NullRetry) Completed() bool {
return true
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.21
require (
github.com/brianvoe/gofakeit/v6 v6.28.0
github.com/go-playground/validator v9.31.0+incompatible
github.com/go-test/deep v1.1.0
github.com/joho/godotenv v1.5.1
github.com/spyzhov/ajson v0.9.1
github.com/stretchr/testify v1.9.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator v9.31.0+incompatible h1:UA72EPEogEnq76ehGdEDp4Mit+3FDh548oRqwVgNsHA=
github.com/go-playground/validator v9.31.0+incompatible/go.mod h1:yrEkQXlcI+PugkyDjY2bRrL/UBU4f3rvrgkN3V8JEig=
github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
Expand Down
23 changes: 22 additions & 1 deletion msdsales/parse.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package msdsales

import (
"strings"

"github.com/amp-labs/connectors/common"
"github.com/spyzhov/ajson"
)
Expand All @@ -19,7 +21,7 @@ func getRecords(node *ajson.Node) ([]map[string]any, error) {
}

func getNextRecordsURL(node *ajson.Node) (string, error) {
return common.JSONManager.GetString(node, "@odata.nextLink")
return common.JSONManager.GetString(node, "@odata.nextLink", true)
}

// FIXME we must differentiate between GET and LIST (it is LIST now)
Expand All @@ -29,7 +31,9 @@ func getNextRecordsURL(node *ajson.Node) (string, error) {
// * hybrid, list of records with extra fields describing list.
func getMarshaledData(records []map[string]interface{}, fields []string) ([]common.ReadResultRow, error) {
data := make([]common.ReadResultRow, len(records))

for i, record := range records {
fields = append(fields, getDisplayFields(fields, record)...)
data[i] = common.ReadResultRow{
Fields: common.ExtractLowercaseFieldsFromRaw(fields, record),
Raw: record,
Expand All @@ -38,3 +42,20 @@ func getMarshaledData(records []map[string]interface{}, fields []string) ([]comm

return data, nil
}

// There are some fields that are only returned in pairs
// these pairs start with the same name and then are followed by @SomeSuffix
// Ex: requesting key `familystatuscode` which is of enum type will include another field defining it as `Married`.
func getDisplayFields(fields []string, payload map[string]any) []string {
displayFields := make([]string, 0)

for _, field := range fields {
for key := range payload {
if strings.HasPrefix(key, field+"@") {
displayFields = append(displayFields, key)
}
}
}

return displayFields
}
3 changes: 2 additions & 1 deletion msdsales/read.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ func (c *Connector) Read(ctx context.Context, config common.ReadParams) (*common
// Next page
fullURL = config.NextPage
}
// TODO given that one of the fields is annotation we can automatically add annotation header (how the hell the end user gonna know about the names of those fields)
// TODO given that one of the fields is annotation we can automatically add annotation header
// (how the hell the end user gonna know about the names of those fields)
rsp, err := c.get(ctx, fullURL, newPaginationHeader(resolvePageSize(config)), annotationsHeader)
if err != nil {
return nil, err
Expand Down
162 changes: 143 additions & 19 deletions msdsales/read_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,16 @@ package msdsales
import (
"context"
"errors"
"github.com/amp-labs/connectors/common"
"github.com/amp-labs/connectors/common/interpreter"
"github.com/amp-labs/connectors/common/reqrepeater"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"testing"

"github.com/amp-labs/connectors/common"
"github.com/amp-labs/connectors/common/interpreter"
"github.com/amp-labs/connectors/common/reqrepeater"
"github.com/go-test/deep"
)

func Test_makeQueryValues(t *testing.T) {
Expand Down Expand Up @@ -59,6 +61,34 @@ func Test_makeQueryValues(t *testing.T) {
}
}

var contactsFirstPageResponse = `{
"@odata.context": "https://org5bd08fdd.api.crm.dynamics.com/api/data/v9.2/$metadata#contacts(fullname,emailaddress1,fax,familystatuscode)",
"@Microsoft.Dynamics.CRM.totalrecordcount": -1,
"@Microsoft.Dynamics.CRM.totalrecordcountlimitexceeded": false,
"@Microsoft.Dynamics.CRM.globalmetadataversion": "6012567",
"value": [
{
"@odata.etag": "W/\"4372108\"",
"fullname": "Heriberto Nathan",
"emailaddress1": "[email protected]",
"fax": "614-555-0122",
"[email protected]": "Single",
"familystatuscode": 1,
"contactid": "cdcfa450-cb0c-ea11-a813-000d3a1b1223"
},
{
"@odata.etag": "W/\"4372115\"",
"fullname": "Dwayne Elijah",
"emailaddress1": "[email protected]",
"fax": "281-555-0158",
"[email protected]": "Single",
"familystatuscode": 1,
"contactid": "9fd4a450-cb0c-ea11-a813-000d3a1b1223"
}
],
"@odata.nextLink": "https://org5bd08fdd.api.crm.dynamics.com/api/data/v9.2/contacts?$select=fullname,emailaddress1,fax,familystatuscode&$skiptoken=%3Ccookie%20pagenumber=%222%22%20pagingcookie=%22%253ccookie%2520page%253d%25221%2522%253e%253ccontactid%2520last%253d%2522%257b9FD4A450-CB0C-EA11-A813-000D3A1B1223%257d%2522%2520first%253d%2522%257bCDCFA450-CB0C-EA11-A813-000D3A1B1223%257d%2522%2520%252f%253e%253c%252fcookie%253e%22%20istracking=%22False%22%20/%3E"
}`

func Test_Read(t *testing.T) {
t.Parallel()

Expand All @@ -84,12 +114,12 @@ func Test_Read(t *testing.T) {
w.WriteHeader(http.StatusBadRequest)
writeBody(w, `{
"error": {
"code": "some-code",
"message": "your fault"
"code": "0x80060888",
"message":"Resource not found for the segment 'conacs'."
}
}`)
})),
expectedErrs: []error{common.ErrBadRequest, errors.New("your fault")},
expectedErrs: []error{common.ErrBadRequest, errors.New("Resource not found for the segment 'conacs'")},
},
{
name: "Incorrect key in payload",
Expand All @@ -113,18 +143,107 @@ func Test_Read(t *testing.T) {
})),
expectedErrs: []error{common.ErrNotArray},
},
// TODO there are more test to write for pagination
//{
// name: "@odata.nextLink must be in payload",
// server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// w.Header().Set("Content-Type", "application/json")
// w.WriteHeader(http.StatusOK)
// writeBody(w, `{
// "value": []
// }`)
// })),
// expectedErrs: []error{common.ErrNotArray},
//},
{
name: "Next page cursor may be missing in payload",
server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
writeBody(w, `{
"value": []
}`)
})),
expected: &common.ReadResult{
Data: []common.ReadResultRow{},
Done: true,
},
expectedErrs: nil,
},
{
name: "Successful read with 2 entries",
server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
writeBody(w, contactsFirstPageResponse)
})),
expected: &common.ReadResult{
Rows: 2,
Data: []common.ReadResultRow{{
Fields: map[string]any{},
Raw: map[string]any{
"@odata.etag": "W/\"4372108\"",
"fullname": "Heriberto Nathan",
"emailaddress1": "[email protected]",
"fax": "614-555-0122",
"[email protected]": "Single",
"familystatuscode": float64(1),
"contactid": "cdcfa450-cb0c-ea11-a813-000d3a1b1223",
},
}, {
Fields: map[string]any{},
Raw: map[string]any{
"@odata.etag": "W/\"4372115\"",
"fullname": "Dwayne Elijah",
"emailaddress1": "[email protected]",
"fax": "281-555-0158",
"[email protected]": "Single",
"familystatuscode": float64(1),
"contactid": "9fd4a450-cb0c-ea11-a813-000d3a1b1223",
},
}},
NextPage: "https://org5bd08fdd.api.crm.dynamics.com/api/data/v9.2/contacts?$select=fullname,emailaddress1,fax,familystatuscode&$skiptoken=%3Ccookie%20pagenumber=%222%22%20pagingcookie=%22%253ccookie%2520page%253d%25221%2522%253e%253ccontactid%2520last%253d%2522%257b9FD4A450-CB0C-EA11-A813-000D3A1B1223%257d%2522%2520first%253d%2522%257bCDCFA450-CB0C-EA11-A813-000D3A1B1223%257d%2522%2520%252f%253e%253c%252fcookie%253e%22%20istracking=%22False%22%20/%3E",
Done: false,
},
expectedErrs: nil,
},
{
// NOTE: all keys in ReadResultRow will be all in lower caps
name: "Successful read with chosen fields plus prepended with display values",
input: common.ReadParams{
Fields: []string{"fullname", "familystatuscode"},
},
server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
writeBody(w, contactsFirstPageResponse)
})),
expected: &common.ReadResult{
Rows: 2,
Data: []common.ReadResultRow{{
Fields: map[string]any{
"fullname": "Heriberto Nathan",
"familystatuscode": float64(1),
"[email protected]": "Single",
},
Raw: map[string]any{
"@odata.etag": "W/\"4372108\"",
"fullname": "Heriberto Nathan",
"emailaddress1": "[email protected]",
"fax": "614-555-0122",
"[email protected]": "Single",
"familystatuscode": float64(1),
"contactid": "cdcfa450-cb0c-ea11-a813-000d3a1b1223",
},
}, {
Fields: map[string]any{
"fullname": "Dwayne Elijah",
"familystatuscode": float64(1),
"[email protected]": "Single",
},
Raw: map[string]any{
"@odata.etag": "W/\"4372115\"",
"fullname": "Dwayne Elijah",
"emailaddress1": "[email protected]",
"fax": "281-555-0158",
"[email protected]": "Single",
"familystatuscode": float64(1),
"contactid": "9fd4a450-cb0c-ea11-a813-000d3a1b1223",
},
}},
NextPage: "https://org5bd08fdd.api.crm.dynamics.com/api/data/v9.2/contacts?$select=fullname,emailaddress1,fax,familystatuscode&$skiptoken=%3Ccookie%20pagenumber=%222%22%20pagingcookie=%22%253ccookie%2520page%253d%25221%2522%253e%253ccontactid%2520last%253d%2522%257b9FD4A450-CB0C-EA11-A813-000D3A1B1223%257d%2522%2520first%253d%2522%257bCDCFA450-CB0C-EA11-A813-000D3A1B1223%257d%2522%2520%252f%253e%253c%252fcookie%253e%22%20istracking=%22False%22%20/%3E",
Done: false,
},
expectedErrs: nil,
},
}

for _, tt := range tests {
Expand Down Expand Up @@ -153,14 +272,19 @@ func Test_Read(t *testing.T) {
t.Fatalf("%s: expected no errors, got: (%v)", tt.name, err)
}

if len(tt.expectedErrs) != 0 && err == nil {
t.Fatalf("%s: expected errors (%v), but got nothing", tt.name, tt.expectedErrs)
}

for _, expectedErr := range tt.expectedErrs {
if !errors.Is(err, expectedErr) && !strings.Contains(err.Error(), expectedErr.Error()) {
t.Fatalf("%s: expected Error: (%v), got: (%v)", tt.name, expectedErr, err)
}
}

if !reflect.DeepEqual(output, tt.expected) {
t.Fatalf("%s: expected: (%v), got: (%v)", tt.name, tt.expected, output)
diff := deep.Equal(output, tt.expected)
t.Fatalf("%s:, \nexpected: (%v), \ngot: (%v), \ndiff: (%v)", tt.name, tt.expected, output, diff)
}
})
}
Expand Down

0 comments on commit 1381b97

Please sign in to comment.