utils.go

// This file is part of arduino-cli.
//
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
//
// This software is released under the GNU General Public License version 3,
// which covers the main part of arduino-cli.
// The terms of this license can be found at:
// https://www.gnu.org/licenses/gpl-3.0.en.html
//
// You can be released from the requirements of the above licenses by purchasing
// a commercial license. Buying such a license is mandatory if you want to
// modify or otherwise use the software for commercial activities involving the
// Arduino software without disclosing the source code of your own applications.
// To purchase a commercial license, send an email to license@arduino.cc.

package utils

import (
"bytes"
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"unicode"
"unicode/utf8"

"github.com/arduino/arduino-cli/i18n"
"github.com/arduino/arduino-cli/legacy/builder/gohasissues"
"github.com/arduino/arduino-cli/legacy/builder/types"
 paths "github.com/arduino/go-paths-helper"
"github.com/pkg/errors"
"golang.org/x/text/runes"
"golang.org/x/text/transform"
"golang.org/x/text/unicode/norm"
)

typefilterFilesfunc([]os.FileInfo) []os.FileInfo

vartr=i18n.Tr

funcReadDirFiltered(folderstring, fnfilterFiles) ([]os.FileInfo, error) {
files, err:=gohasissues.ReadDir(folder)
iferr!=nil {
returnnil, errors.WithStack(err)
 }
returnfn(files), nil
}

funcFilterDirs(files []os.FileInfo) []os.FileInfo {
varfiltered []os.FileInfo
for_, info:=rangefiles {
ifinfo.IsDir() {
filtered=append(filtered, info)
 }
 }
returnfiltered
}

funcFilterFilesWithExtensions(extensions...string) filterFiles {
returnfunc(files []os.FileInfo) []os.FileInfo {
varfiltered []os.FileInfo
for_, file:=rangefiles {
if!file.IsDir() &&SliceContains(extensions, filepath.Ext(file.Name())) {
filtered=append(filtered, file)
 }
 }
returnfiltered
 }
}

funcFilterFiles() filterFiles {
returnfunc(files []os.FileInfo) []os.FileInfo {
varfiltered []os.FileInfo
for_, file:=rangefiles {
if!file.IsDir() {
filtered=append(filtered, file)
 }
 }
returnfiltered
 }
}

varSOURCE_CONTROL_FOLDERS=map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true}

// FilterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name
varFilterOutHiddenFiles=paths.FilterOutPrefixes(".")

// FilterOutSCCS is a ReadDirFilter that excludes known VSC or project files
funcFilterOutSCCS(file*paths.Path) bool {
return!SOURCE_CONTROL_FOLDERS[file.Base()]
}

// FilterReadableFiles is a ReadDirFilter that accepts only readable files
funcFilterReadableFiles(file*paths.Path) bool {
// See if the file is readable by opening it
f, err:=file.Open()
iferr!=nil {
returnfalse
 }
f.Close()
returntrue
}

funcIsSCCSOrHiddenFile(file os.FileInfo) bool {
returnIsSCCSFile(file) ||IsHiddenFile(file)
}

funcIsHiddenFile(file os.FileInfo) bool {
name:=filepath.Base(file.Name())
returnname[0] =='.'
}

funcIsSCCSFile(file os.FileInfo) bool {
name:=filepath.Base(file.Name())
returnSOURCE_CONTROL_FOLDERS[name]
}

funcSliceContains(slice []string, targetstring) bool {
for_, value:=rangeslice {
ifvalue==target {
returntrue
 }
 }
returnfalse
}

typemapFuncfunc(string) string

funcMap(slice []string, fnmapFunc) []string {
newSlice:= []string{}
for_, elem:=rangeslice {
newSlice=append(newSlice, fn(elem))
 }
returnnewSlice
}

typefilterFuncfunc(string) bool

funcFilter(slice []string, fnfilterFunc) []string {
newSlice:= []string{}
for_, elem:=rangeslice {
iffn(elem) {
newSlice=append(newSlice, elem)
 }
 }
returnnewSlice
}

funcWrapWithHyphenI(valuestring) string {
return"\"-I"+value+"\""
}

funcTrimSpace(valuestring) string {
returnstrings.TrimSpace(value)
}

funcprintableArgument(argstring) string {
ifstrings.ContainsAny(arg, "\"\\\t") {
arg=strings.Replace(arg, "\\", "\\\\", -1)
arg=strings.Replace(arg, "\"", "\\\"", -1)
return"\""+arg+"\""
 } else {
returnarg
 }
}

// Convert a command and argument slice back to a printable string.
// This adds basic escaping which is sufficient for debug output, but
// probably not for shell interpretation. This essentially reverses
// ParseCommandLine.
funcPrintableCommand(parts []string) string {
returnstrings.Join(Map(parts, printableArgument), " ")
}

const (
Ignore=0// Redirect to null
Show=1// Show on stdout/stderr as normal
ShowIfVerbose=2// Show if verbose is set, Ignore otherwise
Capture=3// Capture into buffer
)

funcExecCommand(ctx*types.Context, command*exec.Cmd, stdoutint, stderrint) ([]byte, []byte, error) {
ifctx.Verbose {
ctx.Info(PrintableCommand(command.Args))
 }

ifstdout==Capture {
buffer:=&bytes.Buffer{}
command.Stdout=buffer
 } elseifstdout==Show|| (stdout==ShowIfVerbose&&ctx.Verbose) {
ifctx.Stdout!=nil {
command.Stdout=ctx.Stdout
 } else {
command.Stdout=os.Stdout
 }
 }

ifstderr==Capture {
buffer:=&bytes.Buffer{}
command.Stderr=buffer
 } elseifstderr==Show|| (stderr==ShowIfVerbose&&ctx.Verbose) {
ifctx.Stderr!=nil {
command.Stderr=ctx.Stderr
 } else {
command.Stderr=os.Stderr
 }
 }

err:=command.Start()
iferr!=nil {
returnnil, nil, errors.WithStack(err)
 }

err=command.Wait()

varoutbytes, errbytes []byte
ifbuf, ok:=command.Stdout.(*bytes.Buffer); ok {
outbytes=buf.Bytes()
 }
ifbuf, ok:=command.Stderr.(*bytes.Buffer); ok {
errbytes=buf.Bytes()
 }

returnoutbytes, errbytes, errors.WithStack(err)
}

funcAbsolutizePaths(files []string) ([]string, error) {
foridx, file:=rangefiles {
iffile=="" {
continue
 }
absFile, err:=filepath.Abs(file)
iferr!=nil {
returnnil, errors.WithStack(err)
 }
files[idx] =absFile
 }

returnfiles, nil
}

funcFindFilesInFolder(dir*paths.Path, recursebool, extensions []string) (paths.PathList, error) {
fileFilter:=paths.AndFilter(
paths.FilterSuffixes(extensions...),
FilterOutHiddenFiles,
FilterOutSCCS,
paths.FilterOutDirectories(),
FilterReadableFiles,
 )
ifrecurse {
dirFilter:=paths.AndFilter(
FilterOutHiddenFiles,
FilterOutSCCS,
 )
returndir.ReadDirRecursiveFiltered(dirFilter, fileFilter)
 }
returndir.ReadDir(fileFilter)
}

funcAppendIfNotPresent(target []string, elements...string) []string {
for_, element:=rangeelements {
if!SliceContains(target, element) {
target=append(target, element)
 }
 }
returntarget
}

funcMD5Sum(data []byte) string {
md5sumBytes:=md5.Sum(data)
returnhex.EncodeToString(md5sumBytes[:])
}

typeloggerActionstruct {
onlyIfVerbosebool
warnbool
msgstring
}

func (l*loggerAction) Run(ctx*types.Context) error {
if!l.onlyIfVerbose||ctx.Verbose {
ifl.warn {
ctx.Warn(l.msg)
 } else {
ctx.Info(l.msg)
 }
 }
returnnil
}

funcLogIfVerbose(warnbool, msgstring) types.Command {
return&loggerAction{onlyIfVerbose: true, warn: warn, msg: msg}
}

// Returns the given string as a quoted string for use with the C
// preprocessor. This adds double quotes around it and escapes any
// double quotes and backslashes in the string.
funcQuoteCppString(strstring) string {
str=strings.Replace(str, "\\", "\\\\", -1)
str=strings.Replace(str, "\"", "\\\"", -1)
return"\""+str+"\""
}

funcQuoteCppPath(path*paths.Path) string {
returnQuoteCppString(path.String())
}

// Parse a C-preprocessor string as emitted by the preprocessor. This
// is a string contained in double quotes, with any backslashes or
// quotes escaped with a backslash. If a valid string was present at the
// start of the given line, returns the unquoted string contents, the
// remainder of the line (everything after the closing "), and true.
// Otherwise, returns the empty string, the entire line and false.
funcParseCppString(linestring) (string, string, bool) {
// For details about how these strings are output by gcc, see:
// https://github.com/gcc-mirror/gcc/blob/a588355ab948cf551bc9d2b89f18e5ae5140f52c/libcpp/macro.c#L491-L511
// Note that the documentation suggests all non-printable
// characters are also escaped, but the implementation does not
// actually do this. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51259
iflen(line) <1||line[0] !='"' {
return"", line, false
 }

i:=1
res:=""
for {
ifi>=len(line) {
return"", line, false
 }

c, width:=utf8.DecodeRuneInString(line[i:])

switchc {
case'\\':
// Backslash, next character is used unmodified
i+=width
ifi>=len(line) {
return"", line, false
 }
res+=string(line[i])
case'"':
// Quote, end of string
returnres, line[i+width:], true
default:
res+=string(c)
 }

i+=width
 }
}

// Normalizes an UTF8 byte slice
// TODO: use it more often troughout all the project (maybe on logger interface?)
funcNormalizeUTF8(buf []byte) []byte {
t:=transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)
result, _, _:=transform.Bytes(t, buf)
returnresult
}

// CopyFile copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file. The file mode will be copied from the source and
// the copied data is synced/flushed to stable storage.
funcCopyFile(src, dststring) (errerror) {
in, err:=os.Open(src)
iferr!=nil {
return
 }
deferin.Close()

out, err:=os.Create(dst)
iferr!=nil {
return
 }
deferfunc() {
ife:=out.Close(); e!=nil {
err=e
 }
 }()

_, err=io.Copy(out, in)
iferr!=nil {
return
 }

err=out.Sync()
iferr!=nil {
return
 }

si, err:=os.Stat(src)
iferr!=nil {
return
 }
err=os.Chmod(dst, si.Mode())
iferr!=nil {
return
 }

return
}

// CopyDir recursively copies a directory tree, attempting to preserve permissions.
// Source directory must exist, destination directory must *not* exist.
// Symlinks are ignored and skipped.
funcCopyDir(srcstring, dststring, extensions []string) (errerror) {
isAcceptedExtension:=func(extstring) bool {
ext=strings.ToLower(ext)
for_, valid:=rangeextensions {
ifext==valid {
returntrue
 }
 }
returnfalse
 }

src=filepath.Clean(src)
dst=filepath.Clean(dst)

si, err:=os.Stat(src)
iferr!=nil {
returnerr
 }
if!si.IsDir() {
returnfmt.Errorf(tr("source is not a directory"))
 }

_, err=os.Stat(dst)
iferr!=nil&&!os.IsNotExist(err) {
return
 }
iferr==nil {
returnfmt.Errorf(tr("destination already exists"))
 }

err=os.MkdirAll(dst, si.Mode())
iferr!=nil {
return
 }

entries, err:=os.ReadDir(src)
iferr!=nil {
return
 }

for_, dirEntry:=rangeentries {
entry, scopeErr:=dirEntry.Info()
ifscopeErr!=nil {
return
 }

srcPath:=filepath.Join(src, entry.Name())
dstPath:=filepath.Join(dst, entry.Name())

ifentry.IsDir() {
err=CopyDir(srcPath, dstPath, extensions)
iferr!=nil {
return
 }
 } else {
// Skip symlinks.
ifentry.Mode()&os.ModeSymlink!=0 {
continue
 }

if!isAcceptedExtension(filepath.Ext(srcPath)) {
continue
 }

err=CopyFile(srcPath, dstPath)
iferr!=nil {
return
 }
 }
 }

return
}