Commit 510b9200 authored by Ma_124's avatar Ma_124

lots of changes

parent 30517e5a
Pipeline #53294066 failed with stages
in 53 seconds
......@@ -66,11 +66,16 @@ func OFFByName(name string) (*OutputFileFormat, error) {
} else {
return OFFTabSep(args[1:]), nil
}
case "html":
case "html:table+meta":
if len(args) != 1 {
return nil, fmt.Errorf("%q: wrong number of arguments (usage: html)", name)
}
return OFFHtml(), nil
case "test:alpha":
if len(args) != 1 {
return nil, fmt.Errorf("%q: wrong number of arguments (usage: test:alpha)", name)
}
return OFFTestAlphabetical(), nil
default:
return nil, fmt.Errorf("%q: unknown off name", name)
}
......
......@@ -99,7 +99,7 @@ func TestOFFByName_MdTableMetaUnknownNArgs(t *testing.T) {
}
func TestOFFByName_Html(t *testing.T) {
off, err := OFFByName("html")
off, err := OFFByName("html:table+meta")
assert.Equal(t, nil, err)
buf := &strings.Builder{}
......@@ -114,8 +114,8 @@ func TestOFFByName_Html(t *testing.T) {
}
func TestOFFByName_HtmlNArgs(t *testing.T) {
off, err := OFFByName("html 'a'")
assert.Equal(t, "\"html 'a'\": wrong number of arguments (usage: html)", err.Error())
off, err := OFFByName("html:table+meta 'a'")
assert.Equal(t, "\"html:table+meta 'a'\": wrong number of arguments (usage: html)", err.Error())
assert.Equal(t, (*OutputFileFormat)(nil), off)
}
......@@ -158,7 +158,7 @@ func TestIFFByName_ProcessAtArgs(t *testing.T) {
}
defer f.Close()
f.Write([]byte("||"))
_, _ = f.Write([]byte("||"))
iff, err := IFFByName("md:list '@" + f.Name() + "'")
assert.Equal(t, nil, err)
......
......@@ -8,3 +8,10 @@ format="md"
[lists.output]
file="outlist.md"
format="md"
[[lists]]
input-link="inplist"
[lists.output]
file="outlist.html"
format="markdown-html"
......@@ -93,7 +93,10 @@ func mainCompile() {
cfgFile := flag.String("config", "awesome.toml", "path of a config file as described at <https://godoc.org/gitlab.com/Ma_124/awesome-framework/cmd/awesome>")
ghAccessToken := flag.String("gh-token", "", "Generate one at https://github.com/settings/tokens")
ghMaxRequests := flag.Int("gh-max-reqs", 6000, "Only for development.")
flag.Parse(os.Args[2:])
err := flag.Parse(os.Args[2:])
if err != nil {
panic(err)
}
if *ghAccessToken == "" {
*ghAccessToken = os.Getenv("AWESOME_GITHUB_TOKEN")
......@@ -309,7 +312,7 @@ func getOutFFByName(name string) *awesomefw.OutputFileFormat {
case "markdown":
return awesomefw.OFFMarkdown()
case "tabsep":
return awesomefw.OFFTabSep()
return awesomefw.OFFTabSep(awesomefw.NameUrlDescTabSepPattern)
case "md-simple":
fallthrough
case "markdown-simple":
......@@ -333,7 +336,7 @@ func mainGen() {
}
defer f.Close()
f.Write([]byte(`
_, _ = f.Write([]byte(`
image: golang:latest
pages:
......@@ -356,7 +359,7 @@ pages:
}
defer f.Close()
f.Write([]byte(`
_, _ = f.Write([]byte(`
language: go
go:
......@@ -392,7 +395,10 @@ deploy:
func mainInternal() {
flag := flagpkg.NewFlagSet("awesome internal", flagpkg.ExitOnError)
gitlab := flag.Bool("gitlabci", false, "Only for internal use.")
flag.Parse(os.Args[2:])
err := flag.Parse(os.Args[2:])
if err != nil {
panic(err)
}
if *gitlab {
_, err := os.Stat("pre-build.awesome.sh")
......@@ -401,7 +407,10 @@ func mainInternal() {
cmd := exec.Command("bash", "pre-build.awesome.sh")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Run()
err = cmd.Run()
if err != nil {
panic(err)
}
}
os.Args = []string{"", ""} // TODO replace this dirty hack
mainCompile()
......
......@@ -4,7 +4,6 @@ import (
"errors"
"github.com/stretchr/testify/assert"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"strconv"
......@@ -73,9 +72,9 @@ func TestCompileFile_TmpFile(t *testing.T) {
}
defer f.Close()
f.Write([]byte(awesomeAwesomeShortOut[:len(awesomeAwesomeShortOut)-1]))
_, _ = f.Write([]byte(awesomeAwesomeShortOut[:len(awesomeAwesomeShortOut)-1]))
f.Close()
_ = f.Close()
buf := &strings.Builder{}
CompileFile(f.Name(), buf, IFFTabSep(NameUrlDescTabSepPattern), OFFTabSep(NameUrlDescTabSepPattern), &ApisCfg{&ApiCfg{}})
......@@ -97,7 +96,7 @@ func TestCompileFile_UnknownFile(t *testing.T) {
}
func TestCompileFile_NoReadPermFile(t *testing.T) {
fname := filepath.Join(os.TempDir(), "awesome-framework-test-no-read-perm-"+strconv.FormatInt(rand.Int63(), 16)+".txt")
fname := filepath.Join(os.TempDir(), "awesome-framework-test-no-read-perm-"+strconv.FormatInt(randSource.Int63(), 36)+".txt")
defer func() {
r := recover()
......
......@@ -118,7 +118,7 @@ A curated list of awesome curated lists of many topics.
| [awesome-sysadmin](https://github.com/kahun/awesome-sysadmin) | Backups, configuration management, DNS, IMAP/POP3, LDAP, monitoring, SSH, statistics, troubleshooting, virtualization, VPN and more. |
For contributing, [open an issue](https://github.com/emijrp/awesome-awesome/issues) and/or a [pull request](https://github.com/emijrp/awesome-awesome/pulls). Above there are some ideas for missing lists, be bold!
`[1:]))), out.String())
`[1:]), blackfridayOpts)), out.String())
}
func TestCompile_TabSepMarkdownHtml(t *testing.T) {
......@@ -142,7 +142,7 @@ A curated list of awesome curated lists of many topics.
| 7654 | 32 | 1 | 2019-01-20 | [awesome-sysadmin](https://github.com/kahun/awesome-sysadmin) | Backups, configuration management, DNS, IMAP/POP3, LDAP, monitoring, SSH, statistics, troubleshooting, virtualization, VPN and more. |
For contributing, [open an issue](https://github.com/emijrp/awesome-awesome/issues) and/or a [pull request](https://github.com/emijrp/awesome-awesome/pulls). Above there are some ideas for missing lists, be bold!
`[1:]))), out.String())
`[1:]), blackfridayOpts)), out.String())
}
func TestCompile_TabSepHtml(t *testing.T) {
......
package awesomefw
import (
"github.com/BurntSushi/toml"
"os"
"strconv"
"io/ioutil"
)
type cfg2 struct {
Lists []*listCfg2 `toml:"list"`
}
type listCfg2 struct {
InputPath string `toml:"input"`
InputFormat string `toml:"format"`
Outputs []*outputCfg2 `toml:"output"`
}
type outputCfg2 struct {
FilePath string `toml:"file"`
FileFormat string `toml:"format"`
}
func GetApisFromEnv(prefix string) (*ApisCfg, error) {
ghMaxReqs, err := strconv.Atoi(os.Getenv(prefix + "MAX_GITHUB_REQS"))
if err != nil {
return nil, err
}
return &ApisCfg{
&ApiCfg{
os.Getenv(prefix + "GITHUB_TOKEN"),
ghMaxReqs,
},
}, nil
}
func ExecuteConfigFile(f string, apis *ApisCfg) []error {
data, err := ioutil.ReadFile(f)
if err != nil {
return []error{err}
}
return ExecuteConfig(data, apis)
}
func ExecuteConfig(data []byte, apis *ApisCfg) []error {
cfg := &cfg2{}
err := toml.Unmarshal(data, cfg)
if err != nil {
return []error{err}
}
var errs []error
for i, l := range cfg.Lists {
for j, out := range l.Outputs {
errs = append(errs, func(i, j int, l *listCfg2, out *outputCfg2) error {
iff, err := IFFByName(l.InputFormat)
if err != nil {
return err
}
off, err := OFFByName(out.FileFormat)
if err != nil {
return err
}
f, err := os.Create(out.FilePath)
if err != nil {
return err
}
defer f.Close()
CompileFile(l.InputPath, f, iff, off, apis)
return nil
}(i, j, l, out))
}
}
if len(errs) == 0 {
return nil
}
return errs
}
package awesomefw
import (
"testing"
"path/filepath"
"os"
"strconv"
"math/rand"
"github.com/stretchr/testify/assert"
"time"
)
func TestExecuteConfigFile(t *testing.T) {
in1 := writeTmpFile("in1", `
"# Awesome Awesome
"
"A curated list of awesome curated lists of many topics.
"
"- [Awesome Awesome](#awesome-awesome)
" - [Computer management](#computer-management)
"## Computer management
"
awesome-scalability https://github.com/binhnguyennus/awesome-scalability An up-to-date and curated reading list for designing high scalability, high availability, high stabilityback-end systems.
awesome-shell https://github.com/alebcay/awesome-shell Command-line frameworks, toolkits, guides and gizmos.
awesome-sysadmin https://github.com/kahun/awesome-sysadmin Backups, configuration management, DNS, IMAP/POP3, LDAP, monitoring, SSH, statistics, troubleshooting, virtualization, VPN and more.
"
"For contributing, [open an issue](https://github.com/emijrp/awesome-awesome/issues) and/or a [pull request](https://github.com/emijrp/awesome-awesome/pulls). Above there are some ideas for missing lists, be bold!
`[1:])
out11 := tmpFile("out11")
out12 := tmpFile("out12")
in2 := writeTmpFile("in2", `
# Awesome Awesome
A curated list of awesome curated lists of many topics.
- [Awesome Awesome](#awesome-awesome)
- [Computer management](#computer-management)
## Computer management
* [awesome-scalability](https://github.com/binhnguyennus/awesome-scalability) - An up-to-date and curated reading list for designing high scalability, high availability, high stabilityback-end systems.
* [awesome-shell](https://github.com/alebcay/awesome-shell) - Command-line frameworks, toolkits, guides and gizmos.
* [awesome-sysadmin](https://github.com/kahun/awesome-sysadmin) - Backups, configuration management, DNS, IMAP/POP3, LDAP, monitoring, SSH, statistics, troubleshooting, virtualization, VPN and more.
For contributing, [open an issue](https://github.com/emijrp/awesome-awesome/issues) and/or a [pull request](https://github.com/emijrp/awesome-awesome/pulls). Above there are some ideas for missing lists, be bold!
`[1:])
out21 := tmpFile("out21")
cfg := writeTmpFile("cfg", `
#!/usr/bin/env awesome compile -config
[[list]]
input="` + in1 + `"
format="tabsep"
[[list.output]]
file="` + out11 + `"
format="test:alpha"
[[list.output]]
file="` + out12 + `"
format="md:table"
[[list]]
input="` + in2 + `"
format="md:list"
[[list.output]]
file="` + out21 + `""
format="tabsep"
`[1:])
expectedOut11 := ``
expectedOut12 := `
# Awesome Awesome
A curated list of awesome curated lists of many topics.
- [Awesome Awesome](#awesome-awesome)
- [Computer management](#computer-management)
## Computer management
| Name | Desc |
|------|------|
| [awesome-scalability](https://github.com/binhnguyennus/awesome-scalability) | An up-to-date and curated reading list for designing high scalability, high availability, high stabilityback-end systems. |
| [awesome-shell](https://github.com/alebcay/awesome-shell) | Command-line frameworks, toolkits, guides and gizmos. |
| [awesome-sysadmin](https://github.com/kahun/awesome-sysadmin) | Backups, configuration management, DNS, IMAP/POP3, LDAP, monitoring, SSH, statistics, troubleshooting, virtualization, VPN and more. |
For contributing, [open an issue](https://github.com/emijrp/awesome-awesome/issues) and/or a [pull request](https://github.com/emijrp/awesome-awesome/pulls). Above there are some ideas for missing lists, be bold!
`[1:]
expectedOut21 := `
"# Awesome Awesome
"
"A curated list of awesome curated lists of many topics.
"
"- [Awesome Awesome](#awesome-awesome)
" - [Computer management](#computer-management)
"## Computer management
"
awesome-scalability https://github.com/binhnguyennus/awesome-scalability An up-to-date and curated reading list for designing high scalability, high availability, high stabilityback-end systems.
awesome-shell https://github.com/alebcay/awesome-shell Command-line frameworks, toolkits, guides and gizmos.
awesome-sysadmin https://github.com/kahun/awesome-sysadmin Backups, configuration management, DNS, IMAP/POP3, LDAP, monitoring, SSH, statistics, troubleshooting, virtualization, VPN and more.
"
"For contributing, [open an issue](https://github.com/emijrp/awesome-awesome/issues) and/or a [pull request](https://github.com/emijrp/awesome-awesome/pulls). Above there are some ideas for missing lists, be bold!
`[1:]
errs := ExecuteConfigFile(cfg, NewDebugReqsApisCfg())
/*if errs != nil {
buf := strings.Builder{}
for _, err := range errs {
if err == nil {
continue
}
buf.Write([]byte("* " + err.Error() + "\n"))
}
panic(buf.String())
}*/
errs = errs
assert.EqualFileContent(t, out11, expectedOut11)
assert.EqualFileContent(t, out12, expectedOut12)
assert.EqualFileContent(t, out21, expectedOut21)
}
func writeTmpFile(prefix string, content string) string {
path := tmpFile(prefix)
f, err := os.Create(path)
if err != nil {
panic(err)
}
defer f.Close()
f.Write([]byte(content))
return path
}
var randSource = rand.NewSource(time.Now().Unix())
func tmpFile(prefix string) string {
return filepath.Join(os.TempDir(), "awesome-framework-test-exe-cfg-" + prefix + "-"+strconv.FormatInt(randSource.Int63(), 36))
}
......@@ -60,49 +60,49 @@ func OFFMarkdown() *OutputFileFormat {
func writeLiteralMd(out io.Writer, l string, ctx *Ctx) {
ctx.OutFFCtx["writeTblHead"] = true
out.Write([]byte(l))
_, _ = out.Write([]byte(l))
}
func writeSimpleItemMd(out io.Writer, item *Item, ctx *Ctx) {
if ctx.OutFFCtx["writeTblHead"] == true || ctx.OutFFCtx["writeTblHead"] == nil {
ctx.OutFFCtx["writeTblHead"] = false
out.Write([]byte("\n| Name | Desc |\n"))
out.Write([]byte("|------|------|\n"))
_, _ = out.Write([]byte("\n| Name | Desc |\n"))
_, _ = out.Write([]byte("|------|------|\n"))
}
out.Write([]byte("| ["))
out.Write([]byte(item.Name))
out.Write([]byte("]("))
out.Write([]byte(item.Url))
out.Write([]byte(") | "))
out.Write([]byte(item.Desc))
out.Write([]byte(" |\n"))
_, _ = out.Write([]byte("| ["))
_, _ = out.Write([]byte(item.Name))
_, _ = out.Write([]byte("]("))
_, _ = out.Write([]byte(item.Url))
_, _ = out.Write([]byte(") | "))
_, _ = out.Write([]byte(item.Desc))
_, _ = out.Write([]byte(" |\n"))
}
func writeItemMd(out io.Writer, item *Item, ctx *Ctx) {
if ctx.OutFFCtx["writeTblHead"] == true || ctx.OutFFCtx["writeTblHead"] == nil {
ctx.OutFFCtx["writeTblHead"] = false
out.Write([]byte("\n| Stars | Forks | Issues | Last Commit | Name | Desc |\n"))
out.Write([]byte("|-------|-------|--------|-------------|------|------|\n"))
_, _ = out.Write([]byte("\n| Stars | Forks | Issues | Last Commit | Name | Desc |\n"))
_, _ = out.Write([]byte("|-------|-------|--------|-------------|------|------|\n"))
}
meta := FetchMeta(item.Url, ctx)
out.Write([]byte("| "))
out.Write([]byte(meta.Stars()))
out.Write([]byte(" | "))
out.Write([]byte(meta.Forks()))
out.Write([]byte(" | "))
out.Write([]byte(meta.Issues()))
out.Write([]byte(" | "))
out.Write([]byte(meta.LastCommit()))
out.Write([]byte(" | ["))
out.Write([]byte(item.Name))
out.Write([]byte("]("))
out.Write([]byte(item.Url))
out.Write([]byte(") | "))
out.Write([]byte(item.Desc))
out.Write([]byte(" |\n"))
_, _ = out.Write([]byte("| "))
_, _ = out.Write([]byte(meta.Stars()))
_, _ = out.Write([]byte(" | "))
_, _ = out.Write([]byte(meta.Forks()))
_, _ = out.Write([]byte(" | "))
_, _ = out.Write([]byte(meta.Issues()))
_, _ = out.Write([]byte(" | "))
_, _ = out.Write([]byte(meta.LastCommit()))
_, _ = out.Write([]byte(" | ["))
_, _ = out.Write([]byte(item.Name))
_, _ = out.Write([]byte("]("))
_, _ = out.Write([]byte(item.Url))
_, _ = out.Write([]byte(") | "))
_, _ = out.Write([]byte(item.Desc))
_, _ = out.Write([]byte(" |\n"))
}
// Same as OFFSimpleMarkdown but compiled to HTML by gopkg.in/russross/blackfriday.v2
......@@ -128,8 +128,10 @@ func startMdHtml(out io.Writer, ctx *Ctx) {
ctx.OutFFCtx["md-html--writer"] = &strings.Builder{}
}
var blackfridayOpts = blackfriday.WithExtensions(blackfriday.CommonExtensions | blackfriday.AutoHeadingIDs)
func endMdHtml(out io.Writer, ctx *Ctx) {
out.Write(blackfriday.Run([]byte(ctx.OutFFCtx["md-html--writer"].(*strings.Builder).String())))
_, _ = out.Write(blackfriday.Run([]byte(ctx.OutFFCtx["md-html--writer"].(*strings.Builder).String()), blackfridayOpts))
}
func writeLiteralMdHtml(out io.Writer, l string, ctx *Ctx) {
......
......@@ -52,24 +52,34 @@ func FetchMeta(url string, ctx *Ctx) Meta {
url = url[:len(url)-1]
}
// TODO https://godoc.org/gitlab.com
// TODO https://gopkg.in/yaml.v2
// TODO https://gopkg.in/go-yaml/yaml.v2
/* TODO FetchMeta URLs
* https://gopkg.in/yaml.v2
* https://gopkg.in/go-yaml/yaml.v2
* GH Pages
* GL Pages
*/
// === GitHub
if strings.HasPrefix(url, "https://github.com/") {
return fetchGitHub(url[19:], ctx)
} else if strings.HasPrefix(url, "https://godoc.org/github.com/") {
return fetchGitHub(url[29:], ctx)
} else if strings.HasPrefix(url, "http://github.com/") {
return fetchGitHub(url[18:], ctx)
// === GoDoc GitHub
} else if strings.HasPrefix(url, "https://godoc.org/github.com/") {
return fetchGitHub(url[29:], ctx)
} else if strings.HasPrefix(url, "http://godoc.org/github.com/") {
return fetchGitHub(url[28:], ctx)
// === GitLab
} else if strings.HasPrefix(url, "https://gitlab.com/") {
return fetchGitLab(strings.Replace(url[19:], "/", "%2F", -1), ctx)
} else if strings.HasPrefix(url, "http://gitlab.com/") {
return fetchGitLab(strings.Replace(url[18:], "/", "%2F", -1), ctx)
// === GoDoc GitLab
} else if strings.HasPrefix(url, "https://godoc.org/gitlab.com/") {
return fetchGitLab(strings.Replace(url[29:], "/", "%2F", -1), ctx)
} else if strings.HasPrefix(url, "http://godoc.org/gitlab.com/") {
return fetchGitLab(strings.Replace(url[28:], "/", "%2F", -1), ctx)
} else {
// TODO pages
return &naMeta{}
}
}
......@@ -100,9 +110,9 @@ func (m ghMeta) LastCommit() string {
func fetchGitHub(repo string, ctx *Ctx) Meta {
var data []byte
if ctx.fetchMetaGhRequests < ctx.ApisCfg.GitHubCfg.MaxRequests { // TODO
if ctx.fetchMetaGhRequests < ctx.ApisCfg.GitHubCfg.MaxRequests { // TODO fetchMetaGhRequests < MaxRequests
ctx.fetchMetaGhRequests++
resp, err := http.Get("https://api.github.com/repos/" + repo + "?access_token=" + ctx.ApisCfg.GitHubCfg.AccessToken) // TODO
resp, err := http.Get("https://api.github.com/repos/" + repo + "?access_token=" + ctx.ApisCfg.GitHubCfg.AccessToken) // TODO AccessToken
if err != nil {
panic(err)
}
......
......@@ -82,7 +82,7 @@ func TestFetchMeta(t *testing.T) {
{"https://godoc.org/github.com/example/repo", "gh"},
{"http://godoc.org/github.com/example/repo", "gh"},
{"https://gitlab.com/example/repo", "na"}, // TODO
{"https://gitlab.com/example/repo", "na"},
{"http://gitlab.com/example/repo", "na"},
{"http://github.com/example/unknown", "na"},
......
......@@ -103,16 +103,18 @@ func OFFTabSep(fields []string) *OutputFileFormat {
}
func writeLiteralTabSep(out io.Writer, l string, _ *Ctx) {
// TODO literal blocks
// TODO align (once multitab support is added)
// TODO multiple `'`s to `"`
/* TODO TabSep Out
* literal blocks
* align (once multitab support is added)
* multiple `'`s to `"`
*/
if l[len(l)-1] == '\n' {
out.Write([]byte("\""))
out.Write([]byte(l))
_, _ = out.Write([]byte("\""))
_, _ = out.Write([]byte(l))
} else {
out.Write([]byte("'"))
out.Write([]byte(l))
out.Write([]byte("\n"))
_, _ = out.Write([]byte("'"))
_, _ = out.Write([]byte(l))
_, _ = out.Write([]byte("\n"))
}
}
......@@ -121,22 +123,22 @@ func writeItemTabSep(fieldDefs []string) func(out io.Writer, it *Item, _ *Ctx) {
for i, fieldDef := range fieldDefs {
switch fieldDef {
case "name":
out.Write([]byte(it.Name))
_, _ = out.Write([]byte(it.Name))
case "url":
out.Write([]byte(it.Url))
_, _ = out.Write([]byte(it.Url))
case "repo-url":
out.Write([]byte(it.RepoUrl))
_, _ = out.Write([]byte(it.RepoUrl))
case "desc":
out.Write([]byte(it.Desc))
_, _ = out.Write([]byte(it.Desc))
default:
panic(fmt.Sprintf("%v: unknown fieldDef (%q)", it, fieldDef))
}
if i < len(fieldDefs)-1 {
out.Write([]byte("\t"))
_, _ = out.Write([]byte("\t"))
}
}
out.Write([]byte("\n"))
_, _ = out.Write([]byte("\n"))
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment