From 8a8afacf8ec65852435417ed23ac3c384ed454a9 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Tue, 17 Jun 2025 22:35:28 +0200 Subject: [PATCH 01/48] Fix import after fork --- README | 2 +- cmd/cgrep/cgrep.go | 2 +- cmd/cindex/cindex.go | 2 +- cmd/csearch/csearch.go | 4 ++-- go.mod | 2 +- index/write.go | 2 +- regexp/match.go | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README b/README index 65e4141..3fcedc8 100644 --- a/README +++ b/README @@ -7,7 +7,7 @@ see http://swtch.com/~rsc/regexp/regexp4.html. To install: - go get github.com/google/codesearch/cmd/... + go get github.com/freva/codesearch/cmd/... Use "go get -u" to update an existing installation. diff --git a/cmd/cgrep/cgrep.go b/cmd/cgrep/cgrep.go index 5e7404f..746aac5 100644 --- a/cmd/cgrep/cgrep.go +++ b/cmd/cgrep/cgrep.go @@ -11,7 +11,7 @@ import ( "os" "runtime/pprof" - "github.com/google/codesearch/regexp" + "github.com/freva/codesearch/regexp" ) var usageMessage = `usage: cgrep [-c] [-h] [-i] [-l] [-n] [-v] regexp [file...] diff --git a/cmd/cindex/cindex.go b/cmd/cindex/cindex.go index d82bf6c..0479d2b 100644 --- a/cmd/cindex/cindex.go +++ b/cmd/cindex/cindex.go @@ -13,7 +13,7 @@ import ( "runtime/pprof" "slices" - "github.com/google/codesearch/index" + "github.com/freva/codesearch/index" ) var usageMessage = `usage: cindex [-list] [-reset] [-zip] [path...] diff --git a/cmd/csearch/csearch.go b/cmd/csearch/csearch.go index 58af5b8..e3bcfac 100644 --- a/cmd/csearch/csearch.go +++ b/cmd/csearch/csearch.go @@ -14,8 +14,8 @@ import ( "runtime/pprof" "strings" - "github.com/google/codesearch/index" - "github.com/google/codesearch/regexp" + "github.com/freva/codesearch/index" + "github.com/freva/codesearch/regexp" ) var usageMessage = `usage: csearch [-c] [-f fileregexp] [-h] [-i] [-l] [-n] regexp diff --git a/go.mod b/go.mod index dc2e7d2..5a89864 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ -module github.com/google/codesearch +module github.com/freva/codesearch go 1.23 diff --git a/index/write.go b/index/write.go index 185ed23..28ce89b 100644 --- a/index/write.go +++ b/index/write.go @@ -16,7 +16,7 @@ import ( "slices" "strings" - "github.com/google/codesearch/sparse" + "github.com/freva/codesearch/sparse" ) // Index writing. See read.go for details of on-disk format. diff --git a/regexp/match.go b/regexp/match.go index 7a90d1a..c1caaad 100644 --- a/regexp/match.go +++ b/regexp/match.go @@ -17,7 +17,7 @@ import ( "strconv" "strings" - "github.com/google/codesearch/sparse" + "github.com/freva/codesearch/sparse" ) // A matcher holds the state for running regular expression search. From 36098484d3f087fd6bb44f1e5e3620188fdfec87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20Hallingstad?= Date: Wed, 25 May 2022 17:19:18 +0200 Subject: [PATCH 02/48] Update match.go (cherry picked from commit c7e389ad718913bec600871a834ca938c52bb3dd) --- regexp/match.go | 82 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/regexp/match.go b/regexp/match.go index c1caaad..d471ee9 100644 --- a/regexp/match.go +++ b/regexp/match.go @@ -351,6 +351,11 @@ func isWordByte(c int) bool { c == '_' } +type LineHit struct { + Lineno int + Line string +} + // TODO: type Grep struct { Regexp *Regexp // regexp to search for @@ -373,6 +378,21 @@ type Grep struct { PostContext int // number of lines to print before buf []byte + + // Note that there may be several matches per line, but we're ignorant. + MatchedLines []LineHit +} + +func (g *Grep) initForMatching() { + if g.buf == nil { + g.buf = make([]byte, 1<<20) + } + + if g.MatchedLines == nil { + g.MatchedLines = make([]LineHit, 0, 100) + } else { + g.MatchedLines = g.MatchedLines[:0] + } } func (g *Grep) AddFlags() { @@ -414,6 +434,17 @@ func (g *Grep) File(name string) { g.Reader(f, name) } +func (g *Grep) File2(name string) { + f, err := os.Open(name) + if err != nil { + fmt.Fprintf(g.Stderr, "%s\n", err) + g.initForMatching() + return + } + defer f.Close() + g.FindMatches(f, name) +} + var nl = []byte{'\n'} func countNL(b []byte) int { @@ -646,3 +677,54 @@ func chomp(s []byte) []byte { } return s[:i] } + +func (g *Grep) FindMatches(r io.Reader, name string) { + g.initForMatching() + var ( + buf = g.buf[:0] + lineno = 1 + beginText = true + endText = false + ) + for { + n, err := io.ReadFull(r, buf[len(buf):cap(buf)]) + buf = buf[:len(buf)+n] + end := len(buf) + if err == nil { + end = bytes.LastIndex(buf, nl) + 1 + } else { + endText = true + } + chunkStart := 0 + for chunkStart < end { + m1 := g.Regexp.Match(buf[chunkStart:end], beginText, endText) + chunkStart + beginText = false + if m1 < chunkStart { + break + } + g.Match = true + lineStart := bytes.LastIndex(buf[chunkStart:m1], nl) + 1 + chunkStart + lineEnd := m1 + 1 + if lineEnd > end { + lineEnd = end + } + lineno += countNL(buf[chunkStart:lineStart]) + line := buf[lineStart:lineEnd] + g.MatchedLines = append(g.MatchedLines, LineHit{ + Lineno: lineno, + Line: string(line), + }) + lineno++ + chunkStart = lineEnd + } + lineno += countNL(buf[chunkStart:end]) + n = copy(buf, buf[end:]) + buf = buf[:n] + if len(buf) == 0 && err != nil { + if err != io.EOF && err != io.ErrUnexpectedEOF { + fmt.Fprintf(g.Stderr, "%s: %v\n", name, err) + } + break + } + } +} From 0bc752d4b386b6d38e3f8148dbbc2d58024daef4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20Hallingstad?= Date: Mon, 6 Jun 2022 12:54:10 +0200 Subject: [PATCH 03/48] Add -index option to specify the index path (cherry picked from commit 0695b6ae39480711e5f6b486362c3a56bf869127) --- cmd/cindex/cindex.go | 15 +++++++++------ cmd/csearch/csearch.go | 3 ++- index/read.go | 5 ++++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/cmd/cindex/cindex.go b/cmd/cindex/cindex.go index 0479d2b..ff2aab2 100644 --- a/cmd/cindex/cindex.go +++ b/cmd/cindex/cindex.go @@ -16,10 +16,10 @@ import ( "github.com/freva/codesearch/index" ) -var usageMessage = `usage: cindex [-list] [-reset] [-zip] [path...] +var usageMessage = `usage: cindex [-index file] [-list] [-reset] [-zip] [path...] Cindex prepares the trigram index for use by csearch. The index is the -file named by $CSEARCHINDEX, or else $HOME/.csearchindex. +file named by -index, or else $CSEARCHINDEX, or else $HOME/.csearchindex. The simplest invocation is @@ -58,6 +58,7 @@ func usage() { } var ( + indexFlag = flag.String("index", "", "path to index file") listFlag = flag.Bool("list", false, "list indexed paths and exit") resetFlag = flag.Bool("reset", false, "discard existing index") verboseFlag = flag.Bool("verbose", false, "print extra information") @@ -72,8 +73,10 @@ func main() { flag.Usage = usage flag.Parse() + indexpath := index.File(*indexFlag) + if *listFlag { - ix := index.Open(index.File()) + ix := index.Open(index.File(indexpath)) if *checkFlag { if err := ix.Check(); err != nil { log.Fatal(err) @@ -96,12 +99,12 @@ func main() { } if *resetFlag && flag.NArg() == 0 { - os.Remove(index.File()) + os.Remove(index.File(indexpath)) return } var roots []index.Path if flag.NArg() == 0 { - ix := index.Open(index.File()) + ix := index.Open(index.File(indexpath)) roots = slices.Collect(ix.Roots().All()) } else { // Translate arguments to absolute paths so that @@ -117,7 +120,7 @@ func main() { slices.SortFunc(roots, index.Path.Compare) } - master := index.File() + master := index.File(indexpath) if _, err := os.Stat(master); err != nil { // Does not exist. *resetFlag = true diff --git a/cmd/csearch/csearch.go b/cmd/csearch/csearch.go index e3bcfac..91c0a2a 100644 --- a/cmd/csearch/csearch.go +++ b/cmd/csearch/csearch.go @@ -52,6 +52,7 @@ var ( fFlag = flag.String("f", "", "search only files with names matching this regexp") iFlag = flag.Bool("i", false, "case-insensitive search") htmlFlag = flag.Bool("html", false, "print HTML output") + indexFlag = flag.String("index", "", "path to index file") verboseFlag = flag.Bool("verbose", false, "print extra information") bruteFlag = flag.Bool("brute", false, "brute force - search all files in index") cpuProfile = flag.String("cpuprofile", "", "write cpu profile to this file") @@ -109,7 +110,7 @@ func Main() { log.Printf("query: %s\n", q) } - ix := index.Open(index.File()) + ix := index.Open(index.File(*indexFlag)) ix.Verbose = *verboseFlag var post []int if *bruteFlag { diff --git a/index/read.go b/index/read.go index 1c8715b..a3bc00c 100644 --- a/index/read.go +++ b/index/read.go @@ -664,7 +664,10 @@ func mmap(file string) mmapData { // File returns the name of the index file to use. // It is either $CSEARCHINDEX or $HOME/.csearchindex. -func File() string { +func File(file string) string { + if file != "" { + return file + } f := os.Getenv("CSEARCHINDEX") if f != "" { return f From bd8384a4adcd103af41590917b722f23e23ff63f Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Sun, 24 Nov 2024 23:30:56 +0100 Subject: [PATCH 04/48] Create REST API for code search --- cmd/cserver/cserver.go | 201 +++++++++++++++++++ cmd/cserver/rest.go | 446 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 647 insertions(+) create mode 100644 cmd/cserver/cserver.go create mode 100644 cmd/cserver/rest.go diff --git a/cmd/cserver/cserver.go b/cmd/cserver/cserver.go new file mode 100644 index 0000000..37e9cb7 --- /dev/null +++ b/cmd/cserver/cserver.go @@ -0,0 +1,201 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "strconv" + "strings" + + "github.com/freva/codesearch/index" +) + +var usageMessage = `usage: cserver [OPTION...] +Start HTTP server, serving a search and view interface of a source tree. + +Options: + -f FIDX Path to file index made on the paths of SOURCE.* + -index IDX Path to index made by cindex on SOURCE. [CSEARCHINDEX] + -p PORT Port to listen to. [80] + -s SOURCE Path to source directory.* + -t TSFILE Path to timestamp file of the last index update.* + -w STATIC Path to static files to serve (cmd/server/static/).* +*) Option is required. + +WARNING: All files and directories below STATIC and SOURCE are accessible from +the cserver HTTP server. +` + +var INDEX_PATH string + +func usage() { + fmt.Fprintf(os.Stderr, usageMessage) + os.Exit(2) +} + +var ( + fFlag = flag.String("f", "", "Path to file index (required)") + indexFlag = flag.String("index", "", "Path to index file [CSEARCHINDEX]") + pFlag = flag.Int("p", 80, "Port to listen to [80]") + sFlag = flag.String("s", "", "Path to the source tree (required)") + tFlag = flag.String("t", "", "Path to the timestamp file of the last index update (required)") + wFlag = flag.String("w", "", "Path to static files to serve [cmd/cserver/static/]") +) + +type Manifest struct { + Servers []Server + Branches []Branch +} + +type Server struct { + Name string + Url string +} + +type Branch struct { + Server string + Dir string + Repo string + Branch *string +} + +func (s Branch) ResolveServer() Server { + server, ok := SERVERS[s.Server] + if !ok { + log.Print("Failed to find " + s.Server + " in SERVERS") + } + return server +} + +type File struct { + Branch Branch + Relpath string +} + +// Server by server name +var SERVERS map[string]Server + +// Branch by dir +var BRANCHES map[string]Branch + +func readManifest(path string) { + manifestFile, e := os.Open(path) + if e != nil { + log.Fatal("Failed to open " + path) + } + defer manifestFile.Close() + manifestData, e := ioutil.ReadAll(manifestFile) + if e != nil { + log.Fatal("Failed to read " + path) + } + + var manifest Manifest + json.Unmarshal(manifestData, &manifest) + + //fmt.Printf("%q\n", manifest) + SERVERS = make(map[string]Server) + for _, server := range manifest.Servers { + SERVERS[server.Name] = server + } + + BRANCHES = make(map[string]Branch) + for _, branch := range manifest.Branches { + BRANCHES["/"+branch.Dir] = branch + } +} + +// path must be relative to the serving directory (sFlag). +func resolvePath(path string) (*File, error) { + prefix := "" + suffix := "/" + path + for { + var offset = strings.Index(suffix[1:], "/") + 1 + if offset < 1 { + return nil, fmt.Errorf("Failed to find branch for " + path) + } + var name = suffix[:offset] + if len(name) == 0 { + return nil, fmt.Errorf("Found empty component for " + path) + } + prefix += name + suffix = suffix[offset:] + branch, ok := BRANCHES[prefix] + if ok { + return &File{Branch: branch, Relpath: suffix}, nil + } + } +} + +func main() { + flag.Usage = usage + flag.Parse() + + if *fFlag == "" { + log.Fatal("-f is required, see -help for usage") + } + fileIndexFileInfo, e := os.Stat(*fFlag) + if e != nil { + if os.IsNotExist(e) { + log.Fatal("No such index file: " + *fFlag) + } else { + log.Fatal("Failed to stat file: " + *fFlag) + } + } + if !fileIndexFileInfo.Mode().IsRegular() { + log.Fatal("Not an index file: " + *fFlag) + } + + INDEX_PATH = index.File(*indexFlag) + indexfileInfo, e := os.Stat(INDEX_PATH) + if e != nil { + if os.IsNotExist(e) { + log.Fatal("No such index file: " + INDEX_PATH) + } else { + log.Fatal("Failed to stat file: " + INDEX_PATH) + } + } + if !indexfileInfo.Mode().IsRegular() { + log.Fatal("Index file points to a directory: " + INDEX_PATH) + } + + if *sFlag == "" { + log.Fatal("-s is required, see -help for usage") + } + if (*sFlag)[len(*sFlag)-1:] != "/" { + *sFlag += "/" + } + + if *tFlag == "" { + log.Fatal("-t is required, see -help for usage") + } + + if *wFlag == "" { + log.Fatal("-w is required, see -help for usage") + } + sFileInfo, e := os.Stat(*wFlag) + if e != nil { + log.Fatal("Failed to open '" + *wFlag + "'") + } + if !sFileInfo.IsDir() { + log.Fatal("Not a directory: " + *wFlag) + } + sFileInfo, e = os.Stat(*wFlag + "/static") + if e != nil || !sFileInfo.IsDir() { + log.Fatal("Does not look like a path to cmd/cserver/static: " + *wFlag) + } + readManifest(*wFlag + "/static/repos.json") + + http.Handle("/static/", http.FileServer(http.Dir(*wFlag))) + http.HandleFunc("/rest/file", RestFileHandler) + http.HandleFunc("/rest/search", RestSearchHandler) + http.ListenAndServe(":"+strconv.Itoa(*pFlag), nil) + fmt.Println("ListenAndServe returned, exiting process!") +} diff --git a/cmd/cserver/rest.go b/cmd/cserver/rest.go new file mode 100644 index 0000000..aa903d2 --- /dev/null +++ b/cmd/cserver/rest.go @@ -0,0 +1,446 @@ +package main + +import ( + "bufio" + "fmt" + "github.com/freva/codesearch/index" + "github.com/freva/codesearch/regexp" + "log" + "net/http" + "os" + stdregexp "regexp" + "strconv" + "strings" + "unicode" +) + +var escapedChars = map[rune]string{ + '"': "\\\"", + '\\': "\\\\", + '\n': "\\n", + '\r': "\\r", + '\t': "\\t", + '\b': "\\b", + '\f': "\\f", +} + +func RemovePathPrefix(path string) string { + return strings.TrimPrefix(path, *sFlag) +} + +func escapeJsonString(str string) string { + var result string + for _, r := range str { + if escaped, ok := escapedChars[r]; ok { + result += escaped + } else if unicode.IsControl(r) { + result += fmt.Sprintf("\\u%04X", r) + } else { + result += string(r) + } + } + return result +} + +func setHeaders(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET") +} + +func handleError(w http.ResponseWriter, f func() error) { + if err := f(); err != nil { + if len(w.Header()) == 0 { + setHeaders(w) + w.WriteHeader(http.StatusBadRequest) + response := fmt.Sprintf("{\"message\": \"%s\"}", escapeJsonString(err.Error())) + if _, wErr := w.Write([]byte(response)); wErr != nil { + log.Printf("Failed to write error response: %v. Original error: %v", wErr, err) + } + } else { + log.Println("Error:", err) + } + } +} + +func maybeWriteComma(w http.ResponseWriter, shouldWriteComma bool) error { + var err error + if shouldWriteComma { + _, err = w.Write([]byte(",")) + } + return err +} + +func writeJsonFileHeader(w http.ResponseWriter, path string, pathRegex *stdregexp.Regexp) error { + if _, err := w.Write([]byte(fmt.Sprintf("{\"path\":\"%s\"", escapeJsonString(path)))); err != nil { + return err + } + + var resolvedFile, err = resolvePath(path) + if err != nil { + log.Printf("Failed to resolve path %s: %v", path, err) + } else { + branch := "master" + if resolvedFile.Branch.Branch != nil { + branch = *resolvedFile.Branch.Branch + } + url := fmt.Sprintf("%s/%s/blob/%s%s", resolvedFile.Branch.ResolveServer().Url, resolvedFile.Branch.Repo, branch, resolvedFile.Relpath) + url = escapeJsonString(url) + if _, err := w.Write([]byte(fmt.Sprintf(",\"url\":\"%s\"", url))); err != nil { + return err + } + } + + if pathRegex != nil { + matches := pathRegex.FindStringSubmatchIndex(path) + rangeStr := fmt.Sprintf(",\"range\":[%d,%d]", matches[0], matches[1]) + if _, err := w.Write([]byte(rangeStr)); err != nil { + return err + } + } + return nil +} + +func search(w http.ResponseWriter, query string, fileFilter string, excludeFileFilter string, maxHits int, ignoreCase bool) error { + // (?m) => ^ and $ match beginning and end of line, respectively + queryPattern := "(?m)" + query + if ignoreCase { + queryPattern = "(?i)" + queryPattern + } + queryRe, err := regexp.Compile(queryPattern) + if err != nil { + return fmt.Errorf("Bad query regular expression: %w", err) + } + queryStdRe, err := stdregexp.Compile(queryPattern) + if err != nil { + log.Print(err) + } + + var fileRe *regexp.Regexp + var fileStdRe *stdregexp.Regexp + if fileFilter != "" { + filePattern := fileFilter + if ignoreCase { + filePattern = "(?i)" + filePattern + } + + fileRe, err = regexp.Compile(filePattern) + if err != nil { + return fmt.Errorf("Bad file regular expression: %w", err) + } + + fileStdRe, err = stdregexp.Compile(filePattern) + if err != nil { + log.Print(err) + } + } + + var xFileRe *regexp.Regexp + if excludeFileFilter != "" { + excludeFilePattern := excludeFileFilter + if ignoreCase { + excludeFilePattern = "(?i)" + excludeFilePattern + } + + xFileRe, err = regexp.Compile(excludeFilePattern) + if err != nil { + log.Print(err) + return fmt.Errorf("Bad exclude file regular expression: %w", err) + } + } + + q := index.RegexpQuery(queryRe.Syntax) + ix := index.Open(INDEX_PATH) + ix.Verbose = false + var post = ix.PostingQuery(q) + + truncated := false + numHits := 0 + + setHeaders(w) + if _, err := w.Write([]byte("{\"files\":[")); err != nil { + return err + } + + for _, fileId := range post { + if numHits >= maxHits { + truncated = true + break + } + + fullPath := ix.Name(fileId) + path := RemovePathPrefix(fullPath.String()) + + if fileRe != nil { + // Retain only those files matching the file pattern. + if fileRe.MatchString(path, true, true) < 0 { + continue + } + } + + if xFileRe != nil { + // Skip files matching the exclude file pattern. + if xFileRe.MatchString(path, true, true) >= 0 { + continue + } + } + + grep := regexp.Grep{ + Regexp: queryRe, + Stderr: os.Stderr, + } + grep.File2(fullPath) + + if len(grep.MatchedLines) == 0 { + continue + } + + if err := maybeWriteComma(w, numHits > 0); err != nil { + return err + } + if err := writeJsonFileHeader(w, path, fileStdRe); err != nil { + return err + } + if _, err := w.Write([]byte(",\"lines\":[")); err != nil { + return err + } + + for i, hit := range grep.MatchedLines { + if err := maybeWriteComma(w, i > 0); err != nil { + return err + } + escapedLine := escapeJsonString(strings.TrimRight(hit.Line, "\n")) + if _, err := w.Write([]byte(fmt.Sprintf("{\"line\":\"%s\"", escapedLine))); err != nil { + return err + } + + lineMeta := fmt.Sprintf(",\"number\":%d", hit.Lineno) + matches := queryStdRe.FindStringSubmatchIndex(hit.Line) + if matches != nil { + lineMeta += fmt.Sprintf(",\"range\":[%d,%d]}", matches[0], matches[1]) + } + if _, err := w.Write([]byte(lineMeta)); err != nil { + return err + } + + numHits += 1 + if numHits >= maxHits+20 { + truncated = true + break + } + } + + if _, err := w.Write([]byte("]}")); err != nil { + return err + } + } + + _, err = w.Write([]byte(fmt.Sprintf("],\"hits\":%d,\"truncated\":%t}", numHits, truncated))) + return err +} + +func searchFile(w http.ResponseWriter, fileFilter string, excludeFileFilter string, maxHits int, ignoreCase bool) error { + filePattern := "(?m)" + fileFilter + if ignoreCase { + filePattern = "(?i)" + filePattern + } + fileRe, err := regexp.Compile(filePattern) + if err != nil { + return fmt.Errorf("Bad file regular expression: %w", err) + } + + // pattern includes e.g. (?i), which is correct even for plain "regexp" package. + fileStdRe, err := stdregexp.Compile(filePattern) + if err != nil { + log.Print(err) + fileStdRe = nil + } + + var xFileRe *regexp.Regexp + if excludeFileFilter != "" { + xFilePattern := excludeFileFilter + if ignoreCase { + xFilePattern = "(?i)" + xFilePattern + } + xFileRe, err = regexp.Compile(xFilePattern) + if err != nil { + return fmt.Errorf("Bad exclude file regular expression: %w", err) + } + } + + // TODO: Fix this path + idx := index.Open(*fFlag) + idx.Verbose = false + query := index.RegexpQuery(fileRe.Syntax) + var post = idx.PostingQuery(query) + + numHits := 0 + truncated := false + + setHeaders(w) + if _, err := w.Write([]byte("{\"files\":[")); err != nil { + return err + } + + for _, fileId := range post { + if numHits >= maxHits { + truncated = true + break + } + + manifest := idx.Name(fileId) + grep := regexp.Grep{Regexp: fileRe, Stderr: os.Stderr} + // This is no better than just looping through the lines + // of the files and matching (AFAIK), so there's only a + // benefit if we don't traverse through all files: Split + // up the list of paths in many. Too many => I/O bound. + grep.File2(manifest.String()) + + for _, hit := range grep.MatchedLines { + path := hit.Line + if len(path) > 0 && path[len(path)-1] == '\n' { + path = path[:len(path)-1] + } + + if xFileRe != nil && xFileRe.MatchString(path, true, true) >= 0 { + continue + } + + if err := maybeWriteComma(w, numHits > 0); err != nil { + return err + } + if err := writeJsonFileHeader(w, path, fileStdRe); err != nil { + return err + } + if _, err := w.Write([]byte("}")); err != nil { + return err + } + + numHits += 1 + if numHits >= maxHits+10 { + truncated = true + break + } + } + } + + _, err = w.Write([]byte(fmt.Sprintf("],\"hits\":%d,\"truncated\":%t}", numHits, truncated))) + return err +} + +func RestSearchHandler(w http.ResponseWriter, r *http.Request) { + handleError(w, func() error { + if err := r.ParseForm(); err != nil { + return err + } + + query := r.Form.Get("q") + fileFilter := r.Form.Get("f") + excludeFileFilter := r.Form.Get("xf") + ignoreCase := r.Form.Get("i") != "" + maxHitsString := r.Form.Get("n") + maxHits, err := strconv.Atoi(maxHitsString) + if err != nil { + maxHits = defaultMaxHits + } else if maxHits > 1000 { + maxHits = 1000 + } + + if query == "" && fileFilter == "" { + return fmt.Errorf("No query or file filter") + } else if query == "" { + return searchFile(w, fileFilter, excludeFileFilter, maxHits, ignoreCase) + } else { + return search(w, query, fileFilter, excludeFileFilter, maxHits, ignoreCase) + } + }) +} + +type MatchedEntry struct { + Line int + Start int + End int +} + +func restShowFile(w http.ResponseWriter, path string, query string, ignoreCase bool) error { + pattern := query + if ignoreCase { + pattern = "(?i)" + pattern + } + re, err := stdregexp.Compile(pattern) + if err != nil { + return err + } + + file, err := os.Open(*sFlag + path) + if err != nil { + return err + } + defer file.Close() + + setHeaders(w) + if _, err := w.Write([]byte("{\"content\":\"")); err != nil { + return err + } + + i := 1 + var matchedEntries []MatchedEntry + scanner := bufio.NewScanner(file) + // Got this error with a 68kB line: bufio.Scanner: token too long + const maxCapacity = 1024 * 1024 // 1 MB + buf := make([]byte, maxCapacity) + scanner.Buffer(buf, maxCapacity) + for scanner.Scan() { + line := scanner.Text() + if _, err := w.Write([]byte(escapeJsonString(line + "\n"))); err != nil { + return err + } + + matches := re.FindStringSubmatchIndex(line) + if matches != nil { + matchedEntries = append(matchedEntries, MatchedEntry{ + Line: i, + Start: matches[0], + End: matches[1], + }) + } + i = i + 1 + } + + if _, err = w.Write([]byte("\",\"matches\":[")); err != nil { + return err + } + for i, entry := range matchedEntries { + if err := maybeWriteComma(w, i > 0); err != nil { + return err + } + entryStr := fmt.Sprintf("{\"line\":%d,\"range\":[%d,%d]}", entry.Line, entry.Start, entry.End) + if _, err := w.Write([]byte(entryStr)); err != nil { + return err + } + } + + if _, err = w.Write([]byte("]}")); err != nil { + return err + } + + return scanner.Err() +} + +func RestFileHandler(w http.ResponseWriter, request *http.Request) { + handleError(w, func() error { + if err := request.ParseForm(); err != nil { + return err + } + + path := request.Form.Get("p") + query := request.Form.Get("q") + ignoreCase := request.Form.Get("i") != "" + + if strings.Contains(path, "..") { + return fmt.Errorf("Path cannot contain \"..\"") + } + + return restShowFile(w, path, query, ignoreCase) + }) +} From df68d7687532afe3752d5f5e00080220fab2904a Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Sun, 24 Nov 2024 23:32:35 +0100 Subject: [PATCH 05/48] Initial frontend --- frontend/.gitignore | 7 + frontend/.prettierrc | 3 + frontend/eslint.config.mjs | 85 + frontend/index.html | 15 + frontend/package.json | 54 + frontend/postcss.config.mjs | 14 + frontend/src/App/error-boundary.tsx | 44 + frontend/src/App/index.tsx | 26 + frontend/src/App/layout/header.tsx | 74 + frontend/src/App/libs/fetcher.ts | 130 + .../App/libs/use-custom-compare-callback.ts | 19 + frontend/src/App/libs/use-search-query.ts | 29 + frontend/src/App/pages/file/index.tsx | 7 + frontend/src/App/pages/search/index.css | 0 frontend/src/App/pages/search/index.tsx | 65 + frontend/src/App/styles/icons.ts | 3 + frontend/src/App/styles/theme.ts | 77 + frontend/src/index.tsx | 10 + frontend/src/vite-env.d.ts | 1 + frontend/tsconfig.json | 24 + frontend/vite.config.mts | 22 + frontend/yarn.lock | 2761 +++++++++++++++++ 22 files changed, 3470 insertions(+) create mode 100644 frontend/.gitignore create mode 100644 frontend/.prettierrc create mode 100644 frontend/eslint.config.mjs create mode 100755 frontend/index.html create mode 100644 frontend/package.json create mode 100644 frontend/postcss.config.mjs create mode 100644 frontend/src/App/error-boundary.tsx create mode 100755 frontend/src/App/index.tsx create mode 100644 frontend/src/App/layout/header.tsx create mode 100644 frontend/src/App/libs/fetcher.ts create mode 100644 frontend/src/App/libs/use-custom-compare-callback.ts create mode 100644 frontend/src/App/libs/use-search-query.ts create mode 100644 frontend/src/App/pages/file/index.tsx create mode 100644 frontend/src/App/pages/search/index.css create mode 100644 frontend/src/App/pages/search/index.tsx create mode 100644 frontend/src/App/styles/icons.ts create mode 100644 frontend/src/App/styles/theme.ts create mode 100755 frontend/src/index.tsx create mode 100644 frontend/src/vite-env.d.ts create mode 100644 frontend/tsconfig.json create mode 100644 frontend/vite.config.mts create mode 100644 frontend/yarn.lock diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 0000000..1770217 --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1,7 @@ +.DS_STORE +.idea +*.iml +.vscode + +build +node_modules/ diff --git a/frontend/.prettierrc b/frontend/.prettierrc new file mode 100644 index 0000000..92cde39 --- /dev/null +++ b/frontend/.prettierrc @@ -0,0 +1,3 @@ +{ + "singleQuote": true +} \ No newline at end of file diff --git a/frontend/eslint.config.mjs b/frontend/eslint.config.mjs new file mode 100644 index 0000000..a1f8813 --- /dev/null +++ b/frontend/eslint.config.mjs @@ -0,0 +1,85 @@ +import react from 'eslint-plugin-react'; +import typescriptEslint from '@typescript-eslint/eslint-plugin'; +import prettierPlugin from 'eslint-plugin-prettier/recommended'; +import eslintPrettierConfig from 'eslint-config-prettier'; +import pluginReactConfig from 'eslint-plugin-react/configs/recommended.js'; +import globals from 'globals'; +import js from '@eslint/js'; +import tsParser from '@typescript-eslint/parser'; + +export default [ + js.configs.recommended, + prettierPlugin, + pluginReactConfig, + eslintPrettierConfig, + + { + ignores: ['build/*', 'node_modules/*', 'public/*'], + }, + { + plugins: { + react: react, + '@typescript-eslint': typescriptEslint, + }, + + languageOptions: { + globals: { + ...globals.browser, + ...globals.commonjs, + ...globals.jest, + }, + + parser: tsParser, + ecmaVersion: 'latest', + sourceType: 'module', + + parserOptions: { + ecmaFeatures: { + jsx: true, + }, + }, + }, + + settings: { + react: { + version: 'detect', + runtime: 'automatic', + }, + }, + + rules: { + '@typescript-eslint/consistent-type-imports': 'error', + '@typescript-eslint/explicit-module-boundary-types': 'off', + 'no-console': 'error', + '@typescript-eslint/no-unused-vars': [ + 'error', + { + argsIgnorePattern: '^_', + varsIgnorePattern: '^_', + caughtErrorsIgnorePattern: '^_', + }, + ], + + 'no-unused-vars': 'off', + 'prettier/prettier': 'error', + 'react/react-in-jsx-scope': 'off', + + strict: 0, + }, + }, + { + files: ['**/__test__/**'], + languageOptions: { + globals: { + ...globals.node, + }, + }, + }, + { + files: ['**/*.ts', '**/*.tsx', '**/*.mts'], + + rules: { + '@typescript-eslint/explicit-function-return-type': 'error', + }, + }, +]; diff --git a/frontend/index.html b/frontend/index.html new file mode 100755 index 0000000..9870faa --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,15 @@ + + + + + + + Vespa Code Search + + + + +
+ + + diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..6d1fa5b --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,54 @@ +{ + "name": "vespa-code-search", + "version": "0.1.0", + "private": true, + "scripts": { + "dev": "vite dev --port 3000", + "build": "vite build --outDir build/", + "test": "CI=true vitest", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "tsc": "tsc" + }, + "dependencies": { + "@fortawesome/free-brands-svg-icons": "^6", + "@fortawesome/free-regular-svg-icons": "^6", + "@fortawesome/free-solid-svg-icons": "^6", + "@mantine/core": "^7", + "@mantine/form": "^7", + "@mantine/hooks": "^7", + "dayjs": "^1", + "lodash": "^4", + "react": "^18", + "react-dom": "^18", + "react-router-dom": "^6" + }, + "devDependencies": { + "@eslint/js": "^9", + "@types/node": "^22", + "@types/react": "^18", + "@types/react-dom": "^18", + "@typescript-eslint/eslint-plugin": "^8", + "@typescript-eslint/parser": "^8", + "@vitejs/plugin-react": "^4", + "eslint": "^9", + "eslint-config-prettier": "^9", + "eslint-plugin-prettier": "^5", + "eslint-plugin-react": "^7", + "globals": "^15", + "postcss": "^8", + "postcss-preset-mantine": "^1", + "postcss-simple-vars": "^7", + "prettier": "^3", + "typescript": "^5", + "typescript-eslint": "^8", + "vite": "^5", + "vitest": "^2" + }, + "browserslist": [ + ">0.2%", + "not dead", + "not ie <= 11", + "not op_mini all" + ] +} diff --git a/frontend/postcss.config.mjs b/frontend/postcss.config.mjs new file mode 100644 index 0000000..0d6062c --- /dev/null +++ b/frontend/postcss.config.mjs @@ -0,0 +1,14 @@ +export default { + plugins: { + 'postcss-preset-mantine': {}, + 'postcss-simple-vars': { + variables: { + 'mantine-breakpoint-xs': '36em', + 'mantine-breakpoint-sm': '48em', + 'mantine-breakpoint-md': '62em', + 'mantine-breakpoint-lg': '75em', + 'mantine-breakpoint-xl': '88em', + }, + }, + }, +}; diff --git a/frontend/src/App/error-boundary.tsx b/frontend/src/App/error-boundary.tsx new file mode 100644 index 0000000..6c052ad --- /dev/null +++ b/frontend/src/App/error-boundary.tsx @@ -0,0 +1,44 @@ +import type { ErrorInfo, PropsWithChildren, ReactNode } from 'react'; +import { PureComponent } from 'react'; +import { Space, Stack, Text, Title } from '@mantine/core'; + +export class ErrorBoundary extends PureComponent { + state: Readonly<{ error: unknown }>; + constructor(props: PropsWithChildren) { + super(props); + this.state = { error: undefined }; + } + + componentDidCatch(exception: Error, errorInfo: ErrorInfo): void { + const meta = { + location: window.location.href, + time: new Date().toISOString(), + error: { + exception: exception.stack ?? exception.message, + ...errorInfo, + }, + ...exception, + }; + this.setState({ error: meta }); + } + + render(): ReactNode { + if (!this.state.error) return this.props.children; + return ( + + + {/**/} + You encountered a bug + Error details: +