init

.gitignore

+sftper
+vendor

api.go

+package main
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	url_ "net/url"
+	"os"
+	"os/user"
+
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	errBadCommand = errors.New("Invalid command")
+	errBadArgs    = errors.New("Invalid arguments")
+)
+
+type command struct {
+	Name string            `json:"command"`
+	Args map[string]string `json:"args"`
+}
+
+type result struct {
+	Status  string      `json:"status"`
+	Message string      `json:"message"`
+	Data    interface{} `json:"data"`
+}
+
+type sftpAPI struct {
+	url    url_.URL
+	client *sftp.Client
+	cfg    ssh.ClientConfig
+	req    io.Reader
+	resp   io.Writer
+}
+
+func readHeader(r io.Reader) (uint32, error) {
+	buf := make([]byte, 4)
+	if _, err := io.ReadFull(r, buf); err != nil {
+		return 0, err
+	}
+	return binary.BigEndian.Uint32(buf), nil
+}
+
+func readPayload(r io.Reader, num uint32) ([]byte, error) {
+	buf := make([]byte, num)
+	if _, err := io.ReadFull(r, buf); err != nil {
+		return buf, err
+	}
+	return buf, nil
+}
+
+func decodeCommand(buf []byte) (command, error) {
+	command := command{}
+	err := json.Unmarshal(buf, &command)
+	return command, err
+}
+
+func (s sftpAPI) commands() <-chan command {
+	ch := make(chan command)
+
+	go func() {
+		for {
+			num, err := readHeader(s.req)
+			if err != nil {
+				continue
+			}
+			buf, err := readPayload(s.req, num)
+
+			cmd, err := decodeCommand(buf)
+			if err != nil {
+				debug("error decoding command: %s", err)
+				continue
+			}
+			ch <- cmd
+		}
+	}()
+
+	return ch
+}
+
+// args: source, dest as abs paths
+func (s sftpAPI) doPut(source, dest string) error {
+	fout, err := s.client.Create(dest)
+	if err != nil {
+		return errors.Wrapf(err, "can't create %s", dest)
+	}
+	defer fout.Close()
+
+	fin, err := os.Open(source)
+	if err != nil {
+		return errors.Wrapf(err, "can't read %s", source)
+	}
+	defer fin.Close()
+
+	_, err = io.Copy(fout, fin)
+	if err != nil {
+		return errors.Wrapf(err, "upload failed for %s -> %s", source, dest)
+	}
+	return nil
+}
+
+// args: source, dest as abs paths
+func (s sftpAPI) doGet(source, dest string) error {
+	fin, err := s.client.Open(source)
+	if err != nil {
+		return errors.Wrapf(err, "can't read %s", source)
+	}
+	defer fin.Close()
+
+	fout, err := os.Create(dest)
+	if err != nil {
+		return errors.Wrapf(err, "can't create %s", dest)
+	}
+	defer fout.Close()
+
+	_, err = io.Copy(fout, fin)
+	if err != nil {
+		return errors.Wrapf(err, "download failed for %s -> %s", source, dest)
+	}
+	return nil
+}
+
+type stat struct {
+	Name  string `json:"name"`
+	Size  int64  `json:"size"`
+	MTime int64  `json:"mtime"`
+}
+
+// args: abspath glob pattern
+func (s sftpAPI) doListdir(path string) ([]stat, error) {
+	stats := []stat{}
+
+	infos, err := s.client.ReadDir(path)
+	if err != nil {
+		return stats, err
+	}
+
+	for _, st := range infos {
+		stats = append(stats, stat{st.Name(), st.Size(), st.ModTime().Unix()})
+	}
+
+	return stats, nil
+}
+
+// args: abspath
+func (s sftpAPI) doDelete(path string) error {
+	return s.client.Remove(path)
+}
+
+func (s *sftpAPI) ensureConnected() error {
+	if _, err := s.client.Getwd(); err != nil {
+		debug("not connected, connecting: %s", err)
+		return s.connect()
+	}
+	return nil
+}
+
+func (s sftpAPI) doCommand(cmd command) result {
+	zult := result{"ok", "", ""}
+
+	err := s.ensureConnected()
+	if err != nil {
+		zult.Status = "fail"
+		zult.Message = fmt.Sprintf("error connecting: %s", err)
+		return zult
+	}
+
+	switch cmd.Name {
+	case "PUT":
+		err := s.doPut(cmd.Args["source"], cmd.Args["dest"])
+		if err != nil {
+			zult.Status = "error"
+			zult.Message = err.Error()
+		}
+	case "GET":
+		err := s.doGet(cmd.Args["source"], cmd.Args["dest"])
+		if err != nil {
+			zult.Status = "error"
+			zult.Message = err.Error()
+		}
+	case "LISTDIR":
+		matches, err := s.doListdir(cmd.Args["path"])
+		if err != nil {
+			zult.Status = "error"
+			zult.Message = err.Error()
+		}
+		zult.Data = matches
+	case "DELETE":
+		err := s.doDelete(cmd.Args["path"])
+		if err != nil {
+			zult.Status = "error"
+			zult.Message = err.Error()
+		}
+	default:
+		zult.Status = "error"
+		zult.Message = fmt.Sprintf("unknown command: \"%s\"", cmd.Name)
+	}
+	return zult
+}
+
+func (s sftpAPI) writeResult(zult result) error {
+	dat, err := json.Marshal(zult)
+	if err != nil {
+		return err
+	}
+	buf := make([]byte, 4)
+	binary.BigEndian.PutUint32(buf, uint32(len(dat)))
+	buf = append(buf, dat...)
+
+	_, err = s.resp.Write(buf)
+	return err
+}
+
+func (s *sftpAPI) connect() error {
+	debug("connecting to %s", s.url.Host)
+	client, err := ssh.Dial("tcp", s.url.Host, &s.cfg)
+	if err != nil {
+		return err
+	}
+	s.client, err = sftp.NewClient(client)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s sftpAPI) close() {
+	if s.client != nil {
+		s.client.Close()
+	}
+}
+
+func cleanURL(url string) (url_.URL, error) {
+	newURL := url_.URL{
+		Scheme: "sftp",
+	}
+
+	u, err := url_.Parse(url)
+	if len(url) == 0 || err != nil {
+		return newURL, err
+	}
+	newURL.Path = u.Path
+
+	username := u.User.Username()
+	if len(username) == 0 {
+		cu, err := user.Current()
+		if err != nil {
+			panic(errors.Wrapf(err, "can't get current user"))
+		}
+		username = cu.Username
+	}
+	newURL.User = url_.User(username)
+
+	host := u.Hostname()
+	if len(u.Port()) != 0 {
+		host += ":" + u.Port()
+	} else {
+		host += ":22"
+	}
+	newURL.Host = host
+
+	return newURL, nil
+}
+
+func parseServerHostKey(fpath string) (ssh.PublicKey, error) {
+	dat, err := ioutil.ReadFile(fpath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error reading host key %s", fpath)
+	}
+	key, _, _, _, err := ssh.ParseAuthorizedKey(dat)
+	if err != nil {
+		return key, errors.Wrapf(err, "error reading host key %s", fpath)
+	}
+	return key, nil
+}
+
+func parsePrivateKey(fpath string) (ssh.Signer, error) {
+	dat, err := ioutil.ReadFile(fpath)
+	if err != nil {
+		return nil, err
+	}
+	signer, err := ssh.ParsePrivateKey(dat)
+	if err != nil {
+		return nil, err
+	}
+	return signer, nil
+}
+
+func newSftpConfig(user, keyFile, hostKey string) (ssh.ClientConfig, error) {
+	cfg := ssh.ClientConfig{
+		User:            user,
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	}
+	if len(hostKey) > 0 {
+		key, err := parseServerHostKey(hostKey)
+		if err != nil {
+			return cfg, errors.Wrap(err, "parsing server host key")
+		}
+		cfg.HostKeyCallback = ssh.FixedHostKey(key)
+		cfg.HostKeyAlgorithms = []string{key.Type()}
+	}
+
+	signer, err := parsePrivateKey(keyFile)
+	if err != nil {
+		return cfg, errors.Wrap(err, "parsing private key")
+	}
+
+	cfg.Auth = []ssh.AuthMethod{
+		ssh.PublicKeys(signer),
+	}
+
+	return cfg, nil
+}
+
+func newSftpApi(url, privateKey, hostKey string, req io.Reader, resp io.Writer) (sftpAPI, error) {
+	var err error
+
+	sftp := sftpAPI{
+		req:  req,
+		resp: resp,
+	}
+
+	sftp.url, err = cleanURL(url)
+	if err != nil {
+		return sftp, err
+	}
+
+	sftp.cfg, err = newSftpConfig(sftp.url.User.Username(), privateKey, hostKey)
+	if err != nil {
+		return sftp, err
+	}
+
+	return sftp, sftp.connect()
+}

build.sh

+#!/usr/bin/env bash
+set -e
+ver=$(git describe)
+sha=$(git rev-parse HEAD)
+dt="$(date +%Y-%m-%d)T$(date +%H:%M:%S)Z"
+buildStr="v${ver} (${dt})"
+go install -ldflags "-X 'main.buildStr=${buildStr}'"

go.mod

+module gitlab.ssec.wisc.edu/brucef/sftper
+
+go 1.12
+
+require (
+	github.com/pkg/errors v0.8.1
+	github.com/pkg/sftp v1.10.1
+	github.com/spf13/pflag v1.0.5
+	golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7
+)

go.sum

+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fclairamb/ftpserver v0.0.0-20181215165500-058ccd38e144 h1:2JV3ttpI2dtCPvcJ/vOLegrdo/O7XZwLE9YMLOIjYLE=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc=
+github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 h1:0hQKqeLdqlt5iIwVOBErRisrHJAN57yOiPRQItI20fU=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

main.go

+package main
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path"
+
+	"github.com/spf13/pflag"
+)
+
+var (
+	verbose  bool
+	buildStr string
+)
+
+func info(fmt string, args ...interface{}) {
+	log.Printf(fmt, args...)
+}
+
+func debug(fmt string, args ...interface{}) {
+	if verbose {
+		info(fmt, args...)
+	}
+}
+
+func defaultPrivateKey() string {
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		panic("could not determine user home dir")
+	}
+	return path.Join(homeDir, ".ssh/id_rsa")
+}
+
+func main() {
+
+	log.SetOutput(os.Stderr)
+
+	pflag.Usage = func() {
+		fmt.Fprintf(os.Stderr, `%s [options]
+A mini JSON API for performing SFTP commands.
+
+Supported commands read from commands:
+
+    {"command": "PUT", "args": {"source": <path>, "dest": <path>}}
+    {"command": "GET", "args": {"source": <path>, "dest": <path>}}
+    {"command": "LISTDIR", "args": {"path": <path>}}
+        returns [{"name": "<filename>", "size": <bytes>, "mtime": <unixtime>}, ...]
+	{"command": "DELETE", "args": {"path": <path>}}
+
+Responses written to stdout:
+
+    {"status": "(ok|error|fail)", "message": "<err message>", "data": (""|??)}
+
+	where "data" is documented with the command.
+
+Options:
+
+`, os.Args[0])
+		pflag.PrintDefaults()
+
+		fmt.Fprintf(os.Stderr, "\n%s\n", buildStr)
+	}
+
+	var (
+		err    error
+		input  io.Reader
+		source = pflag.StringP("commands", "c", "-", "File where commands are read from.")
+		help   = pflag.Bool("help", false, "Print help and exit")
+		urlArg = pflag.String("url", "", "SFTP base url sftp://[<user>@]<hostname>[:port]. Any provided path will be ignored")
+		// pflag.BoolVar(&verbose, "verbose", false, "Verbose output")
+		pKey = pflag.StringP("pkey", "i", defaultPrivateKey(), "Path to PEM formatted private key.")
+		hKey = pflag.StringP("hkey", "h", "", "Path to PEM formatted host public key. If not "+
+			"provided server host key checking will be disabled. To get a server host key run "+
+			"ssh-keyscan -t rsa <hostname>.")
+	)
+	pflag.BoolVar(&verbose, "verbose", false, "Verbose output to stderr")
+
+	pflag.Parse()
+
+	if *help {
+		pflag.Usage()
+		os.Exit(2)
+	}
+
+	if *source == "-" {
+		input = os.Stdin
+	} else {
+		input, err = os.Open(*source)
+		if err != nil {
+			info("could not open source")
+			os.Exit(2)
+		}
+	}
+
+	sftp, err := newSftpApi(*urlArg, *pKey, *hKey, input, os.Stdout)
+	if err != nil {
+		info("could not initialize api: %s", err)
+		os.Exit(2)
+	}
+	defer sftp.close()
+
+	for cmd := range sftp.commands() {
+		debug("command: %+v", cmd)
+		zult := sftp.doCommand(cmd)
+		debug("result %+v", zult)
+
+		if zult.Status == "fail" {
+			info("FAIL: %+v", zult)
+		}
+		sftp.writeResult(zult)
+	}
+}