Commit 8aca18a3 authored by Lee Brown's avatar Lee Brown

Remove a bunch of invalid country codes for geonames

parent 7ab573c6
......@@ -477,8 +477,8 @@ func NewConfig(log *log.Logger, targetEnv Env, awsCredentials devdeploy.AwsCrede
"s3:ListBucket",
},
Resource: []string{
"arn:aws:s3:::"+cfg.AwsS3BucketPublic.BucketName,
"arn:aws:s3:::"+cfg.AwsS3BucketPrivate.BucketName,
"arn:aws:s3:::" + cfg.AwsS3BucketPublic.BucketName,
"arn:aws:s3:::" + cfg.AwsS3BucketPrivate.BucketName,
},
},
{
......@@ -489,8 +489,8 @@ func NewConfig(log *log.Logger, targetEnv Env, awsCredentials devdeploy.AwsCrede
"s3:GetObject",
},
Resource: []string{
"arn:aws:::"+cfg.AwsS3BucketPublic.BucketName+"/*",
"arn:aws:::"+cfg.AwsS3BucketPrivate.BucketName+"/*",
"arn:aws:::" + cfg.AwsS3BucketPublic.BucketName + "/*",
"arn:aws:::" + cfg.AwsS3BucketPrivate.BucketName + "/*",
},
},
{
......
......@@ -17,9 +17,7 @@ import (
"time"
"geeks-accelerator/oss/saas-starter-kit/internal/platform/web/webcontext"
"github.com/huandu/go-sqlbuilder"
// "github.com/jmoiron/sqlx"
"github.com/pkg/errors"
"github.com/sethgrid/pester"
"github.com/shopspring/decimal"
......@@ -37,15 +35,15 @@ func ValidGeonameCountries(ctx context.Context) []string {
return []string{"US"}
}
return []string{
"AD", "AR", "AS", "AT", "AU", "AX", "BD", "BE", "BG", "BM",
"BR", "BY", "CA", "CH", "CO", "CR", "CZ", "DE", "DK", "DO",
"AD", "AR", "AS", "AT", "AU", "AX", "BD", "BE", "BG",
"BR", "CA", "CH", "CZ", "DE", "DK", "DO",
"DZ", "ES", "FI", "FO", "FR", "GB", "GF", "GG", "GL", "GP",
"GT", "GU", "HR", "HU", "IE", "IM", "IN", "IS", "IT", "JE",
"JP", "LI", "LK", "LT", "LU", "LV", "MC", "MD", "MH", "MK",
"MP", "MQ", "MT", "MX", "MY", "NC", "NL", "NO", "NZ", "PH",
"GT", "GU", "HR", "HU", "IM", "IN", "IS", "IT", "JE",
"JP", "LI", "LK", "LT", "LU", "MC", "MD", "MH", "MK",
"MP", "MQ", "MX", "MY", "NL", "NO", "NZ", "PH",
"PK", "PL", "PM", "PR", "PT", "RE", "RO", "RU", "SE", "SI",
"SJ", "SK", "SM", "TH", "TR", "UA", "US", "UY", "VA", "VI",
"WF", "YT", "ZA"}
"SJ", "SK", "SM", "TH", "TR", "US", "VA", "VI",
"YT", "ZA"}
}
// FindGeonames ....
......
......@@ -245,6 +245,9 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
countries = []string{"US"}
}
fmt.Println("isUnittest", isUnittest)
fmt.Println("countries", countries)
ncol := 12
fn := func(geoNames []geonames.Geoname) error {
valueStrings := make([]string, 0, len(geoNames))
......@@ -282,10 +285,10 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
}
start := time.Now()
for _, country := range countries {
//fmt.Println("LoadGeonames: start country: ", country)
log.Println("LoadGeonames: start country: ", country)
v, err := geoRepo.GetGeonameCountry(context.Background(), country)
if err != nil {
return errors.WithStack(err)
return errors.WithMessagef(err, "Failed to load country %s", country)
}
//fmt.Println("Geoname records: ", len(v))
// Max argument values of Postgres is about 54460. So the batch size for bulk insert is selected 4500*12 (ncol)
......@@ -297,25 +300,25 @@ func migrationList(ctx context.Context, db *sqlx.DB, log *log.Logger, isUnittest
if n == 0 {
err := fn(v)
if err != nil {
return errors.WithStack(err)
return err
}
} else {
for i := 0; i < n; i++ {
vn := v[i*batch : (i+1)*batch]
err := fn(vn)
if err != nil {
return errors.WithStack(err)
return err
}
if n > 0 && n%25 == 0 {
time.Sleep(200)
}
}
if len(v)%batch > 0 {
fmt.Println("Remain part: ", len(v)-n*batch)
log.Printf("Remain part: %d\n", len(v)-n*batch)
vn := v[n*batch:]
err := fn(vn)
if err != nil {
return errors.WithStack(err)
return err
}
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment