pga-create: separate options for discover and repack subcommands so if the dump should be read from stdin you must use --stdin option

Signed-off-by: Manuel Carmona <[email protected]>

Author: mcarmonaa

PublicGitArchive/pga-create/cmd/pga-create/discover.go

@@ -26,14 +26,16 @@ const (
 )
 
 type discoverCommand struct {
-	URL          string `short:"l" long:"url" description:"Link to GHTorrent MySQL dump in tar.gz format. If empty (default), read from stdin if available or find the most recent dump at GHTORRENT_MYSQL ?= http://ghtorrent-downloads.ewi.tudelft.nl/mysql/."`
+	URL          string `short:"l" long:"url" description:"Link to GHTorrent MySQL dump in tar.gz format. If empty (default), it find the most recent dump at GHTORRENT_MYSQL ?= http://ghtorrent-downloads.ewi.tudelft.nl/mysql/."`
+	Stdin        bool   `long:"stdin" description:"read GHTorrent MySQL dump from stdin"`
 	Stars        string `short:"s" long:"stars" default:"data/stars.gz" description:"Output path for the file with the numbers of stars per repository."`
 	Languages    string `short:"g" long:"languages" default:"data/languages.gz" description:"Output path for the gzipped file with the mapping between languages and repositories. May be empty - will be skipped then."`
 	Repositories string `short:"r" long:"repositories" default:"data/repositories.gz" description:"Output path for the gzipped file with the repository names and identifiers."`
 }
 
 func (c *discoverCommand) Execute(args []string) error {
-	discoverRepos(discoveryParameters{
+	discoverRepos(&discoveryParameters{
+		Stdin:         c.Stdin,
 		URL:           c.URL,
 		StarsPath:     c.Stars,
 		LanguagesPath: c.Languages,
@@ -44,6 +46,7 @@ func (c *discoverCommand) Execute(args []string) error {
 }
 
 type discoveryParameters struct {
+	Stdin         bool
 	URL           string
 	StarsPath     string
 	LanguagesPath string
@@ -259,7 +262,7 @@ func findMostRecentMySQLDump(root string) string {
 	return ghturl.ResolveReference(dumpurl).String()
 }
 
-func discoverRepos(params discoveryParameters) {
+func discoverRepos(params *discoveryParameters) {
 	startTime := time.Now()
 
 	for _, p := range []string{
@@ -279,7 +282,7 @@ func discoverRepos(params discoveryParameters) {
 	spin.Start()
 	defer spin.Stop()
 
-	inputFile := dumpReader(params.URL, spin)
+	inputFile := dumpReader(params.Stdin, params.URL, spin)
 	defer inputFile.Close()
 
 	var totalRead int64
@@ -347,8 +350,8 @@ func discoverRepos(params discoveryParameters) {
 		humanize.Bytes(uint64(totalRead)), humanize.Bytes(uint64(processed)), time.Since(startTime))
 }
 
-func dumpReader(url string, spin *spinner.Spinner) io.ReadCloser {
-	if url == "" {
+func dumpReader(stdin bool, url string, spin *spinner.Spinner) io.ReadCloser {
+	if stdin {
 		fi, err := os.Stdin.Stat()
 		if err != nil {
 			fail("checking stat on stdin", err)
@@ -357,7 +360,9 @@ func dumpReader(url string, spin *spinner.Spinner) io.ReadCloser {
 		if fi.Mode()&os.ModeNamedPipe != 0 {
 			return os.Stdin
 		}
+	}
 
+	if url == "" {
 		envURL := os.Getenv("GHTORRENT_MYSQL")
 		if envURL == "" {
 			envURL = defaultGhtorrentMySQL

PublicGitArchive/pga-create/cmd/pga-create/repack.go

@@ -15,23 +15,24 @@ import (
 )
 
 type repackCommand struct {
-	URL    string `short:"l" long:"url" description:"Link to GHTorrent MySQL dump in tar.gz format. If empty (default), read from stdin if possible or find the most recent dump at GHTORRENT_MYSQL ?= http://ghtorrent-downloads.ewi.tudelft.nl/mysql/."`
+	URL    string `short:"l" long:"url" description:"Link to GHTorrent MySQL dump in tar.gz format. If empty (default), it find the most recent dump at GHTORRENT_MYSQL ?= http://ghtorrent-downloads.ewi.tudelft.nl/mysql/."`
+	Stdin  bool   `long:"stdin" description:"read GHTorrent MySQL dump from stdin"`
 	Output string `short:"o" long:"output" required:"true" description:"output file"`
 }
 
 func (c *repackCommand) Execute(args []string) error {
-	repack(c.URL, c.Output)
+	repack(c.Stdin, c.URL, c.Output)
 
 	return nil
 }
 
-func repack(url, output string) {
+func repack(stdin bool, url, output string) {
 	startTime := time.Now()
 	spin := spinner.New(spinner.CharSets[11], 100*time.Millisecond)
 	spin.Start()
 	defer spin.Stop()
 
-	inputFile := dumpReader(url, spin)
+	inputFile := dumpReader(stdin, url, spin)
 	var totalRead int64
 	inputFile = trackingReader{RealReader: inputFile, Callback: func(n int) {
 		totalRead += int64(n)