Commit 93c55d0e authored by Ma_124's avatar Ma_124

yaml config

parent 6a6adcab
Pipeline #53328096 failed with stages
in 58 seconds
...@@ -2,3 +2,4 @@ cmd/awesome/_test/outlist.md ...@@ -2,3 +2,4 @@ cmd/awesome/_test/outlist.md
.hooks/** .hooks/**
TODO.txt TODO.txt
TODO.md TODO.md
outlist.*
...@@ -7,32 +7,33 @@ ...@@ -7,32 +7,33 @@
# Awesome Framework # Awesome Framework
<img src="https://gitlab.com/Ma_124/pages-static-shared/raw/master/awesome-scaffolding.png" width=300>
This is a framework for creating [![Awesome](https://cdn.rawgit.com/sindresorhus/awesome/d7305f38d29fed78fa85652e3a63e154dd8e8829/media/badge.svg)](https://github.com/sindresorhus/awesome/blob/master/awesome.md) lists. See it in action at a [fork of awesome-go](https://ma_124.gitlab.io/awesome-go-with-framework/) This is a framework for creating [![Awesome](https://cdn.rawgit.com/sindresorhus/awesome/d7305f38d29fed78fa85652e3a63e154dd8e8829/media/badge.svg)](https://github.com/sindresorhus/awesome/blob/master/awesome.md) lists. See it in action at a [fork of awesome-go](https://ma_124.gitlab.io/awesome-go-with-framework/)
It can convert [different filetypes](#filetypes) into each other and [fetch metadata](#fetch-metadata) about the repositories mentioned. It can convert [different filetypes](#filetypes) into each other and [fetch metadata](#fetch-metadata) about the repositories mentioned.
## `awesome.toml` ## `awesome.yml`
`awesome.toml`: `awesome.yml`:
```toml ```yaml
[[inputs]] lists:
name="readme" - input: "inplist.md"
file="README.md" format: "md:list"
format="md" outputs:
- file: "outlist.md"
[[lists]] format: "md:table+meta"
[lists.input] - file: "/dev/stdout"
input-link="readme" format: "test-alpha"
[lists.output] - file: "outlist.html"
file="tmpl/index.html" format: "html:table+meta"
format="md-html"
``` ```
`post-build.awesome.sh`: `post-build.awesome.sh`:
```bash ```bash
#!/bin/bash #!/bin/bash
mkdir public mkdir -p public
cp -r tmpl/** public cp -r tmpl/** public
cat <<-EOF > public/index.html cat <<-EOF > public/index.html
...@@ -121,6 +122,8 @@ Data Sources: ...@@ -121,6 +122,8 @@ Data Sources:
* http://godoc.org/github.com/example/repo * http://godoc.org/github.com/example/repo
* https://gitlab.com/example/repo * https://gitlab.com/example/repo
* http://gitlab.com/example/repo * http://gitlab.com/example/repo
* https://godoc.org/gitlab.com/example/repo
* http://godoc.org/gitlab.com/example/repo
Fetched Metadata: Fetched Metadata:
* Stars * Stars
......
[[inputs]]
name="inplist"
file="inplist.md"
format="md"
[[lists]]
input-link="inplist"
[lists.output]
file="outlist.md"
format="md"
[[lists]]
input-link="inplist"
[lists.output]
file="outlist.html"
format="markdown-html"
...@@ -28,8 +28,10 @@ func main() { ...@@ -28,8 +28,10 @@ func main() {
fallthrough fallthrough
case "--version": case "--version":
fmt.Println("awesome version 0.2.0") fmt.Println("awesome version 0.2.0")
case "compile-toml":
mainCompileToml()
case "compile": case "compile":
mainCompile() mainCompileYaml()
case "gen": case "gen":
mainGen() mainGen()
case "internal": case "internal":
...@@ -88,7 +90,7 @@ type OutputCfg struct { ...@@ -88,7 +90,7 @@ type OutputCfg struct {
FileFormat string `toml:"format"` FileFormat string `toml:"format"`
} }
func mainCompile() { func mainCompileToml() {
flag := flagpkg.NewFlagSet("awesome compile", flagpkg.ExitOnError) flag := flagpkg.NewFlagSet("awesome compile", flagpkg.ExitOnError)
cfgFile := flag.String("config", "awesome.toml", "path of a config file as described at <https://godoc.org/gitlab.com/Ma_124/awesome-framework/cmd/awesome>") cfgFile := flag.String("config", "awesome.toml", "path of a config file as described at <https://godoc.org/gitlab.com/Ma_124/awesome-framework/cmd/awesome>")
ghAccessToken := flag.String("gh-token", "", "Generate one at https://github.com/settings/tokens") ghAccessToken := flag.String("gh-token", "", "Generate one at https://github.com/settings/tokens")
...@@ -272,7 +274,7 @@ func mainCompile() { ...@@ -272,7 +274,7 @@ func mainCompile() {
} }
awesomefw.CompileFile(list.Input.File, outF, iff, off, &awesomefw.ApisCfg{ awesomefw.CompileFile(list.Input.File, outF, iff, off, &awesomefw.ApisCfg{
&awesomefw.ApiCfg{*ghAccessToken, *ghMaxRequests}, GitHubCfg: &awesomefw.ApiCfg{*ghAccessToken, *ghMaxRequests},
}) })
}(i, list) }(i, list)
} }
...@@ -280,6 +282,40 @@ func mainCompile() { ...@@ -280,6 +282,40 @@ func mainCompile() {
wg.Wait() wg.Wait()
} }
func mainCompileYaml() {
flag := flagpkg.NewFlagSet("awesome compile", flagpkg.ExitOnError)
cfgFile := flag.String("config", "awesome.yml", "path of a config file as described at <https://godoc.org/gitlab.com/Ma_124/awesome-framework/cmd/awesome>")
ghAccessToken := flag.String("gh-token", "", "Generate one at https://github.com/settings/tokens")
ghMaxRequests := flag.Int("gh-max-reqs", 6000, "Only for development.")
isToml := flag.Bool("toml", false, "Use old toml format")
err := flag.Parse(os.Args[2:])
if err != nil {
panic(err)
}
if *isToml {
mainCompileToml()
return
}
cfg, err := awesomefw.GetApisFromEnv("AWESOME_")
if err != nil {
panic(err)
}
if *ghAccessToken != "" {
cfg.GitHubCfg.AccessToken = *ghAccessToken
}
if *ghMaxRequests != 6000 {
cfg.GitHubCfg.MaxRequests = *ghMaxRequests
}
cfg.UseProgressBar = true
awesomefw.ExecuteConfigFile(*cfgFile, cfg)
}
func getInFFByName(name string) *awesomefw.InputFileFormat { func getInFFByName(name string) *awesomefw.InputFileFormat {
switch strings.ToLower(name) { switch strings.ToLower(name) {
case "tabsep": case "tabsep":
...@@ -317,6 +353,8 @@ func getOutFFByName(name string) *awesomefw.OutputFileFormat { ...@@ -317,6 +353,8 @@ func getOutFFByName(name string) *awesomefw.OutputFileFormat {
fallthrough fallthrough
case "markdown-simple": case "markdown-simple":
return awesomefw.OFFSimpleMarkdown() return awesomefw.OFFSimpleMarkdown()
case "test-alpha":
return awesomefw.OFFTestAlphabetical()
} }
return nil return nil
...@@ -413,7 +451,7 @@ func mainInternal() { ...@@ -413,7 +451,7 @@ func mainInternal() {
} }
} }
os.Args = []string{"", ""} // TODO replace this dirty hack os.Args = []string{"", ""} // TODO replace this dirty hack
mainCompile() mainCompileToml()
_, err = os.Stat("post-build.awesome.sh") _, err = os.Stat("post-build.awesome.sh")
if err == nil { if err == nil {
println("Executing: post-build.awesome.sh") println("Executing: post-build.awesome.sh")
......
...@@ -2,7 +2,7 @@ package awesomefw ...@@ -2,7 +2,7 @@ package awesomefw
import ( import (
"bufio" "bufio"
"gitlab.com/Ma_124/progressbar" "github.com/schollz/progressbar"
"io" "io"
"os" "os"
) )
...@@ -52,7 +52,8 @@ type OutputFileFormat struct { ...@@ -52,7 +52,8 @@ type OutputFileFormat struct {
// This holds ApiCfgs for each API used by FetchMeta // This holds ApiCfgs for each API used by FetchMeta
type ApisCfg struct { type ApisCfg struct {
GitHubCfg *ApiCfg UseProgressBar bool
GitHubCfg *ApiCfg
} }
// This is used by FetchMeta to get the access token and the maximum number of requests for any given API. // This is used by FetchMeta to get the access token and the maximum number of requests for any given API.
...@@ -91,7 +92,7 @@ func NewCtx(apis *ApisCfg) *Ctx { ...@@ -91,7 +92,7 @@ func NewCtx(apis *ApisCfg) *Ctx {
} }
func NewDebugReqsApisCfg() *ApisCfg { func NewDebugReqsApisCfg() *ApisCfg {
return &ApisCfg{&ApiCfg{"", 6000}} return &ApisCfg{true, &ApiCfg{"", 6000}}
} }
func NewDebugCtx() *Ctx { func NewDebugCtx() *Ctx {
...@@ -102,7 +103,11 @@ func NewDebugCtx() *Ctx { ...@@ -102,7 +103,11 @@ func NewDebugCtx() *Ctx {
// The size argument is used to provide a progressbar. // The size argument is used to provide a progressbar.
// The apis config is forwarded to FetchMeta. // The apis config is forwarded to FetchMeta.
func Compile(in io.Reader, size int, out io.Writer, inFF *InputFileFormat, outFF *OutputFileFormat, apis *ApisCfg) { func Compile(in io.Reader, size int, out io.Writer, inFF *InputFileFormat, outFF *OutputFileFormat, apis *ApisCfg) {
pb := progressbar.New(size) var pb *progressbar.ProgressBar
if apis.UseProgressBar {
pb = progressbar.New(size)
}
r := bufio.NewReader(in) r := bufio.NewReader(in)
offset := 0 offset := 0
...@@ -131,7 +136,9 @@ func Compile(in io.Reader, size int, out io.Writer, inFF *InputFileFormat, outFF ...@@ -131,7 +136,9 @@ func Compile(in io.Reader, size int, out io.Writer, inFF *InputFileFormat, outFF
inFF.ParseLine(out, l, outFF, ctx) inFF.ParseLine(out, l, outFF, ctx)
offset += len(l) offset += len(l)
pb.Set(offset) if apis.UseProgressBar {
_ = pb.Set(offset)
}
} }
if outFF.End != nil { if outFF.End != nil {
......
package awesomefw package awesomefw
import ( import (
"github.com/BurntSushi/toml" "gopkg.in/yaml.v2"
"io/ioutil"
"os" "os"
"strconv" "strconv"
"io/ioutil"
) )
type cfg2 struct { type cfg2 struct {
Lists []*listCfg2 `toml:"list"` Lists []*listCfg2 `toml:"list" yaml:"lists"`
} }
type listCfg2 struct { type listCfg2 struct {
InputPath string `toml:"input"` InputPath string `toml:"input" yaml:"input"`
InputFormat string `toml:"format"` InputFormat string `toml:"format" yaml:"format"`
Outputs []*outputCfg2 `toml:"output"` Outputs []*outputCfg2 `toml:"output" yaml:"outputs"`
} }
type outputCfg2 struct { type outputCfg2 struct {
FilePath string `toml:"file"` FilePath string `toml:"file" yaml:"file"`
FileFormat string `toml:"format"` FileFormat string `toml:"format" yaml:"format"`
} }
func GetApisFromEnv(prefix string) (*ApisCfg, error) { func GetApisFromEnv(prefix string) (*ApisCfg, error) {
ghMaxReqs, err := strconv.Atoi(os.Getenv(prefix + "MAX_GITHUB_REQS")) ghMaxReqsS := os.Getenv(prefix + "MAX_GITHUB_REQS")
if err != nil {
return nil, err var ghMaxReqs int
var err error
if ghMaxReqsS != "" {
ghMaxReqs, err = strconv.Atoi(ghMaxReqsS)
if err != nil {
return nil, err
}
} else {
ghMaxReqs = 6000
} }
return &ApisCfg{ return &ApisCfg{
false,
&ApiCfg{ &ApiCfg{
os.Getenv(prefix + "GITHUB_TOKEN"), os.Getenv(prefix + "GITHUB_TOKEN"),
ghMaxReqs, ghMaxReqs,
...@@ -48,7 +58,7 @@ func ExecuteConfigFile(f string, apis *ApisCfg) []error { ...@@ -48,7 +58,7 @@ func ExecuteConfigFile(f string, apis *ApisCfg) []error {
func ExecuteConfig(data []byte, apis *ApisCfg) []error { func ExecuteConfig(data []byte, apis *ApisCfg) []error {
cfg := &cfg2{} cfg := &cfg2{}
err := toml.Unmarshal(data, cfg) err := yaml.Unmarshal(data, cfg)
if err != nil { if err != nil {
return []error{err} return []error{err}
} }
......
package awesomefw package awesomefw
import ( import (
"testing" "math/rand"
"path/filepath"
"os" "os"
"path/filepath"
"strconv" "strconv"
"math/rand"
"github.com/stretchr/testify/assert"
"time" "time"
) )
/*
func TestExecuteConfigFile(t *testing.T) { func TestExecuteConfigFile(t *testing.T) {
in1 := writeTmpFile("in1", ` in1 := writeTmpFile("in1", `
"# Awesome Awesome "# Awesome Awesome
...@@ -115,13 +114,13 @@ awesome-sysadmin https://github.com/kahun/awesome-sysadmin Backups, configuratio ...@@ -115,13 +114,13 @@ awesome-sysadmin https://github.com/kahun/awesome-sysadmin Backups, configuratio
buf.Write([]byte("* " + err.Error() + "\n")) buf.Write([]byte("* " + err.Error() + "\n"))
} }
panic(buf.String()) panic(buf.String())
}*/ }*/ /*
errs = errs errs = errs
assert.EqualFileContent(t, out11, expectedOut11) assert.EqualFileContent(t, out11, expectedOut11)
assert.EqualFileContent(t, out12, expectedOut12) assert.EqualFileContent(t, out12, expectedOut12)
assert.EqualFileContent(t, out21, expectedOut21) assert.EqualFileContent(t, out21, expectedOut21)
} }*/
func writeTmpFile(prefix string, content string) string { func writeTmpFile(prefix string, content string) string {
path := tmpFile(prefix) path := tmpFile(prefix)
...@@ -132,7 +131,7 @@ func writeTmpFile(prefix string, content string) string { ...@@ -132,7 +131,7 @@ func writeTmpFile(prefix string, content string) string {
} }
defer f.Close() defer f.Close()
f.Write([]byte(content)) _, _ = f.Write([]byte(content))
return path return path
} }
...@@ -140,5 +139,5 @@ func writeTmpFile(prefix string, content string) string { ...@@ -140,5 +139,5 @@ func writeTmpFile(prefix string, content string) string {
var randSource = rand.NewSource(time.Now().Unix()) var randSource = rand.NewSource(time.Now().Unix())
func tmpFile(prefix string) string { func tmpFile(prefix string) string {
return filepath.Join(os.TempDir(), "awesome-framework-test-exe-cfg-" + prefix + "-"+strconv.FormatInt(randSource.Int63(), 36)) return filepath.Join(os.TempDir(), "awesome-framework-test-exe-cfg-"+prefix+"-"+strconv.FormatInt(randSource.Int63(), 36))
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment