Reintroduce header patterns for filetype detection

internal/buffer/buffer.go

@@ -693,15 +693,16 @@ func (b *Buffer) UpdateRules() {
 		return
 	}
 
-	// syntaxFileBuffer is a helper structure
+	// syntaxFileInfo is an internal helper structure
 	// to store properties of one single syntax file
-	type syntaxFileBuffer struct {
+	type syntaxFileInfo struct {
 		header    *highlight.Header
 		fileName  string
 		syntaxDef *highlight.Def
 	}
 
-	syntaxFiles := []syntaxFileBuffer{}
+	fnameMatches := []syntaxFileInfo{}
+	headerMatches := []syntaxFileInfo{}
 	syntaxFile := ""
 	foundDef := false
 	var header *highlight.Header
@@ -718,26 +719,46 @@ func (b *Buffer) UpdateRules() {
 			screen.TermMessage("Error parsing header for syntax file " + f.Name() + ": " + err.Error())
 			continue
 		}
-		file, err := highlight.ParseFile(data)
-		if err != nil {
-			screen.TermMessage("Error parsing syntax file " + f.Name() + ": " + err.Error())
-			continue
+
+		matchedFileType := false
+		matchedFileName := false
+		matchedFileHeader := false
+
+		if ft == "unknown" || ft == "" {
+			if header.MatchFileName(b.Path) {
+				matchedFileName = true
+			}
+			if len(fnameMatches) == 0 && header.MatchFileHeader(b.lines[0].data) {
+				matchedFileHeader = true
+			}
+		} else if header.FileType == ft {
+			matchedFileType = true
 		}
 
-		if ((ft == "unknown" || ft == "") && header.MatchFileName(b.Path)) || header.FileType == ft {
+		if matchedFileType || matchedFileName || matchedFileHeader {
+			file, err := highlight.ParseFile(data)
+			if err != nil {
+				screen.TermMessage("Error parsing syntax file " + f.Name() + ": " + err.Error())
+				continue
+			}
+
 			syndef, err := highlight.ParseDef(file, header)
 			if err != nil {
 				screen.TermMessage("Error parsing syntax file " + f.Name() + ": " + err.Error())
 				continue
 			}
-			foundDef = true
 
-			if header.FileType == ft {
+			if matchedFileType {
 				b.SyntaxDef = syndef
 				syntaxFile = f.Name()
+				foundDef = true
 				break
-			} else {
-				syntaxFiles = append(syntaxFiles, syntaxFileBuffer{header, f.Name(), syndef})
+			}
+
+			if matchedFileName {
+				fnameMatches = append(fnameMatches, syntaxFileInfo{header, f.Name(), syndef})
+			} else if matchedFileHeader {
+				headerMatches = append(headerMatches, syntaxFileInfo{header, f.Name(), syndef})
 			}
 		}
 	}
@@ -759,7 +780,10 @@ func (b *Buffer) UpdateRules() {
 
 			if ft == "unknown" || ft == "" {
 				if header.MatchFileName(b.Path) {
-					syntaxFiles = append(syntaxFiles, syntaxFileBuffer{header, f.Name(), nil})
+					fnameMatches = append(fnameMatches, syntaxFileInfo{header, f.Name(), nil})
+				}
+				if len(fnameMatches) == 0 && header.MatchFileHeader(b.lines[0].data) {
+					headerMatches = append(headerMatches, syntaxFileInfo{header, f.Name(), nil})
 				}
 			} else if header.FileType == ft {
 				syntaxFile = f.Name()
@@ -769,33 +793,49 @@ func (b *Buffer) UpdateRules() {
 	}
 
 	if syntaxFile == "" {
-		length := len(syntaxFiles)
+		matches := fnameMatches
+		if len(matches) == 0 {
+			matches = headerMatches
+		}
+
+		length := len(matches)
 		if length > 0 {
 			signatureMatch := false
 			if length > 1 {
+				// multiple matching syntax files found, try to resolve the ambiguity
+				// using signatures
 				detectlimit := util.IntOpt(b.Settings["detectlimit"])
 				lineCount := len(b.lines)
 				limit := lineCount
 				if detectlimit > 0 && lineCount > detectlimit {
 					limit = detectlimit
 				}
-				for i := 0; i < length && !signatureMatch; i++ {
-					if syntaxFiles[i].header.HasFileSignature() {
-						for j := 0; j < limit && !signatureMatch; j++ {
-							if syntaxFiles[i].header.MatchFileSignature(b.lines[j].data) {
-								syntaxFile = syntaxFiles[i].fileName
-								b.SyntaxDef = syntaxFiles[i].syntaxDef
-								header = syntaxFiles[i].header
+
+			matchLoop:
+				for _, m := range matches {
+					if m.header.HasFileSignature() {
+						for i := 0; i < limit; i++ {
+							if m.header.MatchFileSignature(b.lines[i].data) {
+								syntaxFile = m.fileName
+								if m.syntaxDef != nil {
+									b.SyntaxDef = m.syntaxDef
+									foundDef = true
+								}
+								header = m.header
 								signatureMatch = true
+								break matchLoop
 							}
 						}
 					}
 				}
 			}
 			if length == 1 || !signatureMatch {
-				syntaxFile = syntaxFiles[0].fileName
-				b.SyntaxDef = syntaxFiles[0].syntaxDef
-				header = syntaxFiles[0].header
+				syntaxFile = matches[0].fileName
+				if matches[0].syntaxDef != nil {
+					b.SyntaxDef = matches[0].syntaxDef
+					foundDef = true
+				}
+				header = matches[0].header
 			}
 		}
 	}

pkg/highlight/parser.go

@@ -39,13 +39,15 @@ type Def struct {
 type Header struct {
 	FileType       string
 	FileNameRegex  *regexp.Regexp
+	HeaderRegex    *regexp.Regexp
 	SignatureRegex *regexp.Regexp
 }
 
 type HeaderYaml struct {
 	FileType string `yaml:"filetype"`
 	Detect   struct {
 		FNameRegexStr     string `yaml:"filename"`
+		HeaderRegexStr    string `yaml:"header"`
 		SignatureRegexStr string `yaml:"signature"`
 	} `yaml:"detect"`
 }
@@ -96,18 +98,22 @@ func init() {
 // A yaml file might take ~400us to parse while a header file only takes ~20us
 func MakeHeader(data []byte) (*Header, error) {
 	lines := bytes.Split(data, []byte{'\n'})
-	if len(lines) < 3 {
+	if len(lines) < 4 {
 		return nil, errors.New("Header file has incorrect format")
 	}
 	header := new(Header)
 	var err error
 	header.FileType = string(lines[0])
 	fnameRegexStr := string(lines[1])
-	signatureRegexStr := string(lines[2])
+	headerRegexStr := string(lines[2])
+	signatureRegexStr := string(lines[3])
 
 	if fnameRegexStr != "" {
 		header.FileNameRegex, err = regexp.Compile(fnameRegexStr)
 	}
+	if err == nil && headerRegexStr != "" {
+		header.HeaderRegex, err = regexp.Compile(headerRegexStr)
+	}
 	if err == nil && signatureRegexStr != "" {
 		header.SignatureRegex, err = regexp.Compile(signatureRegexStr)
 	}
@@ -134,6 +140,9 @@ func MakeHeaderYaml(data []byte) (*Header, error) {
 	if hdrYaml.Detect.FNameRegexStr != "" {
 		header.FileNameRegex, err = regexp.Compile(hdrYaml.Detect.FNameRegexStr)
 	}
+	if err == nil && hdrYaml.Detect.HeaderRegexStr != "" {
+		header.HeaderRegex, err = regexp.Compile(hdrYaml.Detect.HeaderRegexStr)
+	}
 	if err == nil && hdrYaml.Detect.SignatureRegexStr != "" {
 		header.SignatureRegex, err = regexp.Compile(hdrYaml.Detect.SignatureRegexStr)
 	}
@@ -154,6 +163,14 @@ func (header *Header) MatchFileName(filename string) bool {
 	return false
 }
 
+func (header *Header) MatchFileHeader(firstLine []byte) bool {
+	if header.HeaderRegex != nil {
+		return header.HeaderRegex.Match(firstLine)
+	}
+
+	return false
+}
+
 // HasFileSignature checks the presence of a stored signature
 func (header *Header) HasFileSignature() bool {
 	return header.SignatureRegex != nil

runtime/help/colors.md

@@ -271,13 +271,40 @@ detect:
 ```
 
 Micro will match this regex against a given filename to detect the filetype.
-You may also provide an optional `signature` regex that will check a certain
-amount of lines of a file to find specific marks. For example:
+
+In addition to the `filename` regex (or even instead of it) you can provide
+a `header` regex that will check the first line of the line. For example:
 
 ```
 detect:
     filename: "\\.ya?ml$"
-    signature: "%YAML"
+    header: "%YAML"
+```
+
+This is useful in cases when the given file name is not sufficient to determine
+the filetype, e.g. with the above example, if a YAML file has no `.yaml`
+extension but may contain a `%YAML` directive in its first line.
+
+`filename` takes precedence over `header`, i.e. if there is a syntax file that
+matches the file with a filetype by the `filename` and another syntax file that
+matches the same file with another filetype by the `header`, the first filetype
+will be used.
+
+Finally, in addition to `filename` and/or `header` (but not instead of them)
+you may also provide an optional `signature` regex which is useful for resolving
+ambiguities when there are multiple syntax files matching the same file with
+different filetypes. If a `signature` regex is given, micro will match a certain
+amount of first lines in the file (this amount is determined by the `detectlimit`
+option) against this regex, and if any of the lines match, this syntax file's
+filetype will be preferred over other matching filetypes.
+
+For example, to distinguish C++ header files from C and Objective-C header files
+that have the same `.h` extension:
+
+```
+detect:
+    filename: "\\.c(c|pp|xx)$|\\.h(h|pp|xx)?$"
+    signature: "namespace|template|public|protected|private"
 ```
 
 ### Syntax rules

runtime/syntax/PowerShell.yaml

@@ -5,7 +5,7 @@ filetype: powershell
 
 detect:
     filename: "\\.ps(1|m1|d1)$"
-    #signature: ""
+    #header: ""
 
 rules:
     # - comment.block:           # Block Comment

runtime/syntax/README.md

@@ -2,7 +2,8 @@
 
 Here are micro's syntax files.
 
-Each yaml file specifies how to detect the filetype based on file extension or given signature. The signature can be matched to all available lines of the file or to the value defined with the option `detectlimit` (to limit parse times) for a best "guess".
+Each yaml file specifies how to detect the filetype based on file extension or header (first line of the line).
+In addition, a signature can be provided to help resolving ambiguities when multiple matching filetypes are detected.
 Then there are patterns and regions linked to highlight groups which tell micro how to highlight that filetype.
 
 Making your own syntax files is very simple. I recommend you check the file after you are finished with the

runtime/syntax/awk.yaml

@@ -2,7 +2,7 @@ filetype: awk
 
 detect:
     filename: "\\.awk$"
-    signature: "^#!.*bin/(env +)?awk( |$)"
+    header: "^#!.*bin/(env +)?awk( |$)"
 
 rules:
     - preproc: "\\$[A-Za-z0-9_!@#$*?\\-]+"

runtime/syntax/bat.yaml

@@ -2,7 +2,7 @@ filetype: batch
 
 detect:
   filename: "(\\.bat$|\\.cmd$)"
-  # signature: ""
+  # header: ""
 
 rules:
   # Numbers

runtime/syntax/crontab.yaml

@@ -2,7 +2,7 @@ filetype: crontab
 
 detect:
     filename: "crontab$"
-    signature: "^#.*?/etc/crontab"
+    header: "^#.*?/etc/crontab"
 
 rules:
       #              The time and date fields are:

runtime/syntax/csx.yaml

@@ -1,7 +1,7 @@
 filetype: csharp-script
 detect:
     filename: "\\.csx$"
-    signature: "^#!.*/(env +)?dotnet-script( |$)"
+    header: "^#!.*/(env +)?dotnet-script( |$)"
 
 rules:
     - include: "csharp"

runtime/syntax/fish.yaml

@@ -2,7 +2,7 @@ filetype: fish
 
 detect:
     filename: "\\.fish$"
-    signature: "^#!.*/(env +)?fish( |$)"
+    header: "^#!.*/(env +)?fish( |$)"
 
 rules:
       # Numbers

runtime/syntax/godoc.yaml

@@ -5,7 +5,7 @@ filetype: godoc
 
 detect:
     filename: "\\.godoc$"
-    signature: package.*import
+    header: package.*import
 
 rules:
     - preproc: "^[^ ].*"

runtime/syntax/groovy.yaml

@@ -2,7 +2,7 @@ filetype: groovy
 
 detect:
     filename: "(\\.(groovy|gy|gvy|gsh|gradle)$|^[Jj]enkinsfile$)"
-    signature: "^#!.*/(env +)?groovy *$"
+    header: "^#!.*/(env +)?groovy *$"
 
 rules:
     # And the style guide for constants is CONSTANT_CASE

runtime/syntax/html4.yaml

@@ -2,7 +2,7 @@ filetype: html4
 
 detect:
     filename: "\\.htm[l]?4$"
-    signature: "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN|http://www.w3.org/TR/html4/strict.dtd\">"
+    header: "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN|http://www.w3.org/TR/html4/strict.dtd\">"
 
 rules:
     - error: "<[^!].*?>"

runtime/syntax/html5.yaml

@@ -2,7 +2,7 @@ filetype: html5
 
 detect:
     filename: "\\.htm[l]?5$"
-    signature: "<!DOCTYPE html5>"
+    header: "<!DOCTYPE html5>"
 
 rules:
     - error: "<[^!].*?>"

runtime/syntax/javascript.yaml

@@ -2,7 +2,7 @@ filetype: javascript
 
 detect:
     filename: "(\\.js$|\\.es[5678]?$|\\.mjs$)"
-    signature: "^#!.*/(env +)?node( |$)"
+    header: "^#!.*/(env +)?node( |$)"
 
 rules:
     - constant.number: "\\b[-+]?([1-9][0-9]*|0[0-7]*|0x[0-9a-fA-F]+)([uU][lL]?|[lL][uU]?)?\\b"

runtime/syntax/json.yaml

@@ -2,7 +2,7 @@ filetype: json
 
 detect:
     filename: "\\.json$"
-    signature: "^\\{$"
+    header: "^\\{$"
 
 rules:
     - constant.number: "\\b[-+]?([1-9][0-9]*|0[0-7]*|0x[0-9a-fA-F]+)([uU][lL]?|[lL][uU]?)?\\b"

runtime/syntax/julia.yaml

@@ -2,7 +2,7 @@ filetype: julia
 
 detect:
     filename: "\\.jl$"
-    signature: "^#!.*/(env +)?julia( |$)"
+    header: "^#!.*/(env +)?julia( |$)"
 
 rules:
 

runtime/syntax/justfile.yaml

@@ -3,7 +3,7 @@ filetype: 'justfile'
 
 detect:
     filename: "(^\\.?[Jj]ustfile|\\.just)$"
-    signature: "^#!.*/(env +)?[bg]?just --justfile"
+    header: "^#!.*/(env +)?[bg]?just --justfile"
 
 rules:
     - preproc: "\\<(ifeq|ifdef|ifneq|ifndef|else|endif)\\>"