Commit 93c55d0e authored by Ma_124's avatar Ma_124

yaml config

parent 6a6adcab
Pipeline #53328096 failed with stages
in 58 seconds
......@@ -2,3 +2,4 @@ cmd/awesome/_test/outlist.md
.hooks/**
TODO.txt
TODO.md
outlist.*
......@@ -7,32 +7,33 @@
# Awesome Framework
<img src="https://gitlab.com/Ma_124/pages-static-shared/raw/master/awesome-scaffolding.png" width=300>
This is a framework for creating [![Awesome](https://cdn.rawgit.com/sindresorhus/awesome/d7305f38d29fed78fa85652e3a63e154dd8e8829/media/badge.svg)](https://github.com/sindresorhus/awesome/blob/master/awesome.md) lists. See it in action at a [fork of awesome-go](https://ma_124.gitlab.io/awesome-go-with-framework/)
It can convert [different filetypes](#filetypes) into each other and [fetch metadata](#fetch-metadata) about the repositories mentioned.
## `awesome.toml`
`awesome.toml`:
```toml
[[inputs]]
name="readme"
file="README.md"
format="md"
[[lists]]
[lists.input]
input-link="readme"
[lists.output]
file="tmpl/index.html"
format="md-html"
## `awesome.yml`
`awesome.yml`:
```yaml
lists:
- input: "inplist.md"
format: "md:list"
outputs:
- file: "outlist.md"
format: "md:table+meta"
- file: "/dev/stdout"
format: "test-alpha"
- file: "outlist.html"
format: "html:table+meta"
```
`post-build.awesome.sh`:
```bash
#!/bin/bash
mkdir public
mkdir -p public
cp -r tmpl/** public
cat <<-EOF > public/index.html
......@@ -121,6 +122,8 @@ Data Sources:
* http://godoc.org/github.com/example/repo
* https://gitlab.com/example/repo
* http://gitlab.com/example/repo
* https://godoc.org/gitlab.com/example/repo
* http://godoc.org/gitlab.com/example/repo
Fetched Metadata:
* Stars
......
[[inputs]]
name="inplist"
file="inplist.md"
format="md"
[[lists]]
input-link="inplist"
[lists.output]
file="outlist.md"
format="md"
[[lists]]
input-link="inplist"
[lists.output]
file="outlist.html"
format="markdown-html"
......@@ -28,8 +28,10 @@ func main() {
fallthrough
case "--version":
fmt.Println("awesome version 0.2.0")
case "compile-toml":
mainCompileToml()
case "compile":
mainCompile()
mainCompileYaml()
case "gen":
mainGen()
case "internal":
......@@ -88,7 +90,7 @@ type OutputCfg struct {
FileFormat string `toml:"format"`
}
func mainCompile() {
func mainCompileToml() {
flag := flagpkg.NewFlagSet("awesome compile", flagpkg.ExitOnError)
cfgFile := flag.String("config", "awesome.toml", "path of a config file as described at <https://godoc.org/gitlab.com/Ma_124/awesome-framework/cmd/awesome>")
ghAccessToken := flag.String("gh-token", "", "Generate one at https://github.com/settings/tokens")
......@@ -272,7 +274,7 @@ func mainCompile() {
}
awesomefw.CompileFile(list.Input.File, outF, iff, off, &awesomefw.ApisCfg{
&awesomefw.ApiCfg{*ghAccessToken, *ghMaxRequests},
GitHubCfg: &awesomefw.ApiCfg{*ghAccessToken, *ghMaxRequests},
})
}(i, list)
}
......@@ -280,6 +282,40 @@ func mainCompile() {
wg.Wait()
}
func mainCompileYaml() {
flag := flagpkg.NewFlagSet("awesome compile", flagpkg.ExitOnError)
cfgFile := flag.String("config", "awesome.yml", "path of a config file as described at <https://godoc.org/gitlab.com/Ma_124/awesome-framework/cmd/awesome>")
ghAccessToken := flag.String("gh-token", "", "Generate one at https://github.com/settings/tokens")
ghMaxRequests := flag.Int("gh-max-reqs", 6000, "Only for development.")
isToml := flag.Bool("toml", false, "Use old toml format")
err := flag.Parse(os.Args[2:])
if err != nil {
panic(err)
}
if *isToml {
mainCompileToml()
return
}
cfg, err := awesomefw.GetApisFromEnv("AWESOME_")
if err != nil {
panic(err)
}
if *ghAccessToken != "" {
cfg.GitHubCfg.AccessToken = *ghAccessToken
}
if *ghMaxRequests != 6000 {
cfg.GitHubCfg.MaxRequests = *ghMaxRequests
}
cfg.UseProgressBar = true
awesomefw.ExecuteConfigFile(*cfgFile, cfg)
}
func getInFFByName(name string) *awesomefw.InputFileFormat {
switch strings.ToLower(name) {
case "tabsep":
......@@ -317,6 +353,8 @@ func getOutFFByName(name string) *awesomefw.OutputFileFormat {
fallthrough
case "markdown-simple":
return awesomefw.OFFSimpleMarkdown()
case "test-alpha":
return awesomefw.OFFTestAlphabetical()
}
return nil
......@@ -413,7 +451,7 @@ func mainInternal() {
}
}
os.Args = []string{"", ""} // TODO replace this dirty hack
mainCompile()
mainCompileToml()
_, err = os.Stat("post-build.awesome.sh")
if err == nil {
println("Executing: post-build.awesome.sh")
......
......@@ -2,7 +2,7 @@ package awesomefw
import (
"bufio"
"gitlab.com/Ma_124/progressbar"
"github.com/schollz/progressbar"
"io"
"os"
)
......@@ -52,7 +52,8 @@ type OutputFileFormat struct {
// This holds ApiCfgs for each API used by FetchMeta
type ApisCfg struct {
GitHubCfg *ApiCfg
UseProgressBar bool
GitHubCfg *ApiCfg
}
// This is used by FetchMeta to get the access token and the maximum number of requests for any given API.
......@@ -91,7 +92,7 @@ func NewCtx(apis *ApisCfg) *Ctx {
}
func NewDebugReqsApisCfg() *ApisCfg {
return &ApisCfg{&ApiCfg{"", 6000}}
return &ApisCfg{true, &ApiCfg{"", 6000}}
}
func NewDebugCtx() *Ctx {
......@@ -102,7 +103,11 @@ func NewDebugCtx() *Ctx {
// The size argument is used to provide a progressbar.
// The apis config is forwarded to FetchMeta.
func Compile(in io.Reader, size int, out io.Writer, inFF *InputFileFormat, outFF *OutputFileFormat, apis *ApisCfg) {
pb := progressbar.New(size)
var pb *progressbar.ProgressBar
if apis.UseProgressBar {
pb = progressbar.New(size)
}
r := bufio.NewReader(in)
offset := 0
......@@ -131,7 +136,9 @@ func Compile(in io.Reader, size int, out io.Writer, inFF *InputFileFormat, outFF
inFF.ParseLine(out, l, outFF, ctx)
offset += len(l)
pb.Set(offset)
if apis.UseProgressBar {
_ = pb.Set(offset)
}
}
if outFF.End != nil {
......
package awesomefw
import (
"github.com/BurntSushi/toml"
"gopkg.in/yaml.v2"
"io/ioutil"
"os"
"strconv"
"io/ioutil"
)
type cfg2 struct {
Lists []*listCfg2 `toml:"list"`
Lists []*listCfg2 `toml:"list" yaml:"lists"`
}
type listCfg2 struct {
InputPath string `toml:"input"`
InputFormat string `toml:"format"`
Outputs []*outputCfg2 `toml:"output"`
InputPath string `toml:"input" yaml:"input"`
InputFormat string `toml:"format" yaml:"format"`
Outputs []*outputCfg2 `toml:"output" yaml:"outputs"`
}
type outputCfg2 struct {
FilePath string `toml:"file"`
FileFormat string `toml:"format"`
FilePath string `toml:"file" yaml:"file"`
FileFormat string `toml:"format" yaml:"format"`
}
func GetApisFromEnv(prefix string) (*ApisCfg, error) {
ghMaxReqs, err := strconv.Atoi(os.Getenv(prefix + "MAX_GITHUB_REQS"))
if err != nil {
return nil, err
ghMaxReqsS := os.Getenv(prefix + "MAX_GITHUB_REQS")
var ghMaxReqs int
var err error
if ghMaxReqsS != "" {
ghMaxReqs, err = strconv.Atoi(ghMaxReqsS)
if err != nil {
return nil, err
}
} else {
ghMaxReqs = 6000
}
return &ApisCfg{
false,
&ApiCfg{
os.Getenv(prefix + "GITHUB_TOKEN"),
ghMaxReqs,
......@@ -48,7 +58,7 @@ func ExecuteConfigFile(f string, apis *ApisCfg) []error {
func ExecuteConfig(data []byte, apis *ApisCfg) []error {
cfg := &cfg2{}
err := toml.Unmarshal(data, cfg)
err := yaml.Unmarshal(data, cfg)
if err != nil {
return []error{err}
}
......
package awesomefw
import (
"testing"
"path/filepath"
"math/rand"
"os"
"path/filepath"
"strconv"
"math/rand"
"github.com/stretchr/testify/assert"
"time"
)
/*
func TestExecuteConfigFile(t *testing.T) {
in1 := writeTmpFile("in1", `
"# Awesome Awesome
......@@ -115,13 +114,13 @@ awesome-sysadmin https://github.com/kahun/awesome-sysadmin Backups, configuratio
buf.Write([]byte("* " + err.Error() + "\n"))
}
panic(buf.String())
}*/
}*/ /*
errs = errs
assert.EqualFileContent(t, out11, expectedOut11)
assert.EqualFileContent(t, out12, expectedOut12)
assert.EqualFileContent(t, out21, expectedOut21)
}
}*/
func writeTmpFile(prefix string, content string) string {
path := tmpFile(prefix)
......@@ -132,7 +131,7 @@ func writeTmpFile(prefix string, content string) string {
}
defer f.Close()
f.Write([]byte(content))
_, _ = f.Write([]byte(content))
return path
}
......@@ -140,5 +139,5 @@ func writeTmpFile(prefix string, content string) string {
var randSource = rand.NewSource(time.Now().Unix())
func tmpFile(prefix string) string {
return filepath.Join(os.TempDir(), "awesome-framework-test-exe-cfg-" + prefix + "-"+strconv.FormatInt(randSource.Int63(), 36))
return filepath.Join(os.TempDir(), "awesome-framework-test-exe-cfg-"+prefix+"-"+strconv.FormatInt(randSource.Int63(), 36))
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment