But there's only one type of string

They were being highlighted after underscores in variable names, and underscores weren't being catered for in non-base-ten numeric literals. Now, fixed!

README.md

@@ -1,6 +1,8 @@
 BBEdit language module for Rust
 ===============================
 
+!["A screenshot of the language module"](https://raw.githubusercontent.com/ogham/Rust.bblm/master/screenshot.png)
+
 This is a BBEdit 11 Language Module for [Rust](http://www.rust-lang.org). It provides the following features:
 
 - Complete syntax highlighting
@@ -15,7 +17,7 @@ By default, it highlights anything beginning with a capital letter in a certain
 
 ### Installation
 
-The simplest way is to just download the package and put it in this folder:
+The simplest way is to just [download the package](https://github.com/ogham/Rust.bblm/releases/tag/0.2) and put it in this folder:
 
     ~/Application Support/BBEdit/Language Modules
 
@@ -24,4 +26,3 @@ Then restart your BBEdit and it should be picked up.
 ### Compilation
 
 To compile your own version, you'll need Xcode. The default schema outputs a `.bblm`. You'll also need the BBEdit SDK. The project assumes it's mounted under `/Volumes/BBEdit SDK`.
-

info.plist

@@ -43,17 +43,17 @@
 				<string>&apos;lifetime</string>
 				<key>me.bsago.bblm.rust.identifier</key>
 				<string>Identifier</string>
-				<key>me.bsago.bblm.rust.attribute</key>
-				<string>#[attribute]</string>
 				<key>me.bsago.bblm.rust.function</key>
-				<string>fn function</string>
+				<string>fn functionname</string>
+				<key>me.bsago.bblm.rust.module</key>
+				<string>mod modulename</string>
 			</dict>
 			<key>BBLMRunColors</key>
 			<dict>
+				<key>me.bsago.bblm.rust.module</key>
+				<string>rgb(135, 2, 219)</string>
 				<key>me.bsago.bblm.rust.lifetime</key>
 				<string>rgb(133, 20, 75)</string>
-				<key>me.bsago.bblm.rust.attribute</key>
-				<string>rgb(177, 13, 201)</string>
 				<key>me.bsago.bblm.rust.identifier</key>
 				<string>rgb(61, 153, 112)</string>
 				<key>me.bsago.bblm.rust.function</key>
@@ -66,15 +66,21 @@
 					<string>com.barebones.bblm.keyword</string>
 					<key>Keywords</key>
 					<array>
+						<string>abstract</string>
+						<string>alignof</string>
 						<string>as</string>
+						<string>become</string>
 						<string>box</string>
 						<string>break</string>
+						<string>const</string>
 						<string>continue</string>
 						<string>crate</string>
+						<string>do</string>
 						<string>else</string>
 						<string>enum</string>
 						<string>extern</string>
 						<string>false</string>
+						<string>final</string>
 						<string>fn</string>
 						<string>for</string>
 						<string>if</string>
@@ -82,25 +88,36 @@
 						<string>in</string>
 						<string>let</string>
 						<string>loop</string>
-						<string>macro_rules!</string>
+						<string>macro</string>
+						<string>macro_rules</string>
 						<string>match</string>
 						<string>mod</string>
+						<string>move</string>
 						<string>mut</string>
+						<string>offsetof</string>
+						<string>override</string>
 						<string>priv</string>
 						<string>proc</string>
 						<string>pub</string>
+						<string>pure</string>
 						<string>ref</string>
 						<string>return</string>
 						<string>self</string>
+						<string>sizeof</string>
 						<string>static</string>
 						<string>struct</string>
 						<string>super</string>
 						<string>trait</string>
 						<string>true</string>
 						<string>type</string>
+						<string>typeof</string>
 						<string>unsafe</string>
+						<string>unsized</string>
 						<string>use</string>
+						<string>virtual</string>
+						<string>where</string>
 						<string>while</string>
+						<string>yield</string>
 					</array>
 				</dict>
 				<dict>
@@ -108,37 +125,22 @@
 					<string>com.barebones.bblm.predefined-symbol</string>
 					<key>Keywords</key>
 					<array>
-						<string>assert</string>
 						<string>bool</string>
 						<string>char</string>
-						<string>debug</string>
-						<string>env</string>
-						<string>error</string>
-						<string>externfn</string>
 						<string>f32</string>
 						<string>f64</string>
-						<string>fail</string>
 						<string>float</string>
-						<string>fmt</string>
 						<string>i16</string>
 						<string>i32</string>
 						<string>i64</string>
 						<string>i8</string>
-						<string>include</string>
+						<string>isize</string>
-						<string>include_bin</string>
-						<string>include_str</string>
-						<string>info</string>
-						<string>int</string>
-						<string>proto</string>
 						<string>str</string>
-						<string>stringify</string>
 						<string>u16</string>
 						<string>u32</string>
 						<string>u64</string>
 						<string>u8</string>
-						<string>uint</string>
+						<string>usize</string>
-						<string>vec</string>
-						<string>warn</string>
 					</array>
 				</dict>
 			</array>

rust.mm

@@ -8,12 +8,12 @@
 #include "BBLMInterface.h"
 #include "BBLMTextIterator.h"
 
-#define kMaxLineLength	256
+#define kMaxLineLength  256
 
 static NSString* const identifierColour = @"me.bsago.bblm.rust.identifier";
-static NSString* const attributeColour = @"me.bsago.bblm.rust.attribute";
 static NSString* const lifetimeColour = @"me.bsago.bblm.rust.lifetime";
 static NSString* const functionColour = @"me.bsago.bblm.rust.function";
+static NSString* const moduleColour = @"me.bsago.bblm.rust.module";
 
 static bool addRun(NSString *kind, int  start,int len , const BBLMCallbackBlock& bblm_callbacks)
 {
@@ -30,13 +30,21 @@ static bool addRun(NSString *kind, int  start,int len , const BBLMCallbackBlock&
 SInt32 skipString(BBLMTextIterator &iter)
 {
     SInt32 length = 1;
-    UniChar terminator = iter.GetNextChar();
     UniChar ch;
     
+    if (iter.strcmp("\"", 1) == 0)
+    {
+        iter += 1;
+    }
+    else
+    {
+        return 0;
+    }
+
     while ((ch = iter.GetNextChar()))
     {
         length++;
-        if (ch == terminator)
+        if (ch == '"')
         {
             break;
         }
@@ -51,6 +59,40 @@ SInt32 skipString(BBLMTextIterator &iter)
     return length;
 }
 
+SInt32 skipRawString(BBLMTextIterator &iter)
+{
+    SInt32 length = 4;
+    UniChar ch;
+    
+    if (iter.strcmp("r##\"", 4) == 0)
+    {
+        iter += 4;
+    }
+    else
+    {
+        return 0;
+    }
+    
+    while ((ch = iter.GetNextChar()))
+    {
+        length++;
+        if (iter.strcmp("\"##", 3) == 0)
+        {
+            length += 3;
+            iter += 3;
+            break;
+        }
+        
+        if (ch == '\\')
+        {
+            iter++;
+            length++;
+        }
+    }
+    
+    return length;
+}
+
 SInt32 skipLineComment(BBLMTextIterator &iter)
 {
     SInt32 length = 2;
@@ -118,7 +160,7 @@ SInt32 skipWord(BBLMTextIterator &iter)
 
     while ((ch = iter.GetNextChar()))
     {
-        if (isalpha(ch) || ch == '_')
+        if (isalpha(ch) || ch == '_' || (length > 0 && isdigit(ch)))
         {
             length++;
         }
@@ -134,13 +176,37 @@ SInt32 skipWord(BBLMTextIterator &iter)
 
 SInt32 skipAttribute(BBLMTextIterator &iter)
 {
-    SInt32 length = 1;
+    SInt32 length = 0;
     UniChar ch;
 
-    iter++;
+    if (iter.strcmp("#[", 2) == 0)
+    {
+        length += 2;
+        iter += 2;
+    }
+    else if (iter.strcmp("#![", 3) == 0)
+    {
+        length += 3;
+        iter += 3;
+    }
+    else if (iter.strcmp("#", 1) == 0)
+    {
+        iter++;
+        return 1;
+    }
+    else
+    {
+        return 0;
+    }
+    
     while ((ch = iter.GetNextChar()))
     {
-        if (ch == '\n' || ch == '\r')
+        if (ch == ']')
+        {
+            length++;
+            break;
+        }
+        else if (ch == '\n' || ch == '\r')
         {
             break;
         }
@@ -156,19 +222,70 @@ SInt32 skipAttribute(BBLMTextIterator &iter)
 SInt32 skipNumber(BBLMTextIterator &iter)
 {
     UInt32 length = 0;
-    UniChar ch;
+    UniChar ch = iter.GetNextChar();
     bool hasSuffix = false;
+    int base = 10;
+
+    if (ch == '0')
+    {
+        ch = iter.GetNextChar();
+        if (ch == 'x')
+        {
+            base = 16;
+            length += 2;
+        }
+        else if (ch == 'b')
+        {
+            base = 2;
+            length += 2;
+        }
+        else if (ch == 'o')
+        {
+            base = 8;
+            length += 2;
+        }
+        else if (ch)
+        {
+            length++;
+            iter--;
+        }
+    }
+    else if (ch)
+    {
+        iter--;
+    }
 
     while ((ch = iter.GetNextChar()))
     {
-        if (isdigit(ch) || (ch == '_' && length > 0))
+        if ((base == 10) && (isdigit(ch) || ((ch == '_' || ch == '.') && length > 0)))
+        {
+            length++;
+        }
+        else if ((base == 2) && (ch == '0' || ch == '1' || ((ch == '_' || ch == '.') && length > 0)))
+        {
+            length++;
+        }
+        else if ((base == 8) && ((ch >= '0' && ch <= '7') || ((ch == '_' || ch == '.') && length > 0)))
+        {
+            length++;
+        }
+        else if ((base == 16) && ((ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F') || isdigit(ch) || ((ch == '_' || ch == '.') && length > 0)))
         {
             length++;
         }
         else if (ch == 'f' || ch == 'u' || ch == 'i')
         {
-            hasSuffix = true;
             length++;
+            if (ch != 'f' && iter.strcmp("s", 1) == 0)
+            {
+                // Parse 'us' or 'is' machine-dependent suffixes
+                length++;
+            }
+            else
+            {
+                // Otherwise, allow for numbers at the end
+                hasSuffix = true;
+            }
             break;
         }
         else
@@ -233,6 +350,14 @@ SInt32 skipToEndOfFunction(BBLMTextIterator &iter)
                 if (braceLevel < 1) return length;
                 break;
 
+            case ';':
+                // If the definition just ends with a semicolon, then it's
+                // either a function in a trait definition, or a C function
+                // definition in an extern, neither of which we want in the
+                // function list.
+                if (braceLevel < 1) return 0;
+                break;
+
             case '(':
                 parenLevel++;
                 break;
@@ -269,17 +394,33 @@ SInt32 scanForSymbol(BBLMTextIterator &iter,
         iter += keywordLen;
         if ((whitespaceLen = skipWhitespace(iter)))
         {
+            bool is_test = iter.strcmp("test", 4) == 0;
+            
             if ((wordLen = skipWord(iter)))
             {
                 UInt32 funLen = skipToEndOfFunction(iter);
+                
+                // Skip over trait method definitions and extern functions
+                if (funLen == 0)
+                {
+                    return 0;
+                }
+                
+                // Ignore modules called 'test'
+                if (strcmp(keyword, "mod") == 0 && is_test)
+                {
+                    return 0;
+                }
+
                 UInt32 tokenOffset, funIndex;
                 UInt32 nameLen;
                 BBLMProcInfo info;
 
                 iter -= (wordLen + funLen);
-                nameLen = keywordLen + whitespaceLen + wordLen;
                 iter -= (keywordLen + whitespaceLen);
 
+                nameLen = keywordLen + whitespaceLen + wordLen;
+
                 bblmAddTokenToBuffer(callbacks, params.fFcnParams.fTokenBuffer, iter.Address(),
                                      nameLen, &tokenOffset);
 
@@ -296,7 +437,7 @@ SInt32 scanForSymbol(BBLMTextIterator &iter,
                 info.fNameStart   = tokenOffset;
                 info.fNameLength  = nameLen;
                 bblmAddFunctionToList(callbacks, params.fFcnParams.fFcnList, info, &funIndex);
-                
+                bblmAddFoldRange(callbacks, info.fFunctionStart, funLen, kBBLMFunctionAutoFold);
                 iter += (keywordLen + whitespaceLen);
                 return info.fFunctionEnd;
             }
@@ -428,7 +569,20 @@ OSErr calculateRuns(BBLMParamBlock &params, const BBLMCallbackBlock *callbacks)
     bool wordchr = false;
     while ((ch = iter.GetNextChar()))
     {
-        if (ch == '"')
+        if (ch == 'r' && iter.strcmp("##\"", 3) == 0)
+        {
+            iter--;
+            if (!makeCodeRun(iter, runStart, *callbacks)) return noErr;
+            runStart = iter.Offset();
+            runLen = skipRawString(iter);
+            
+            // Use the heredoc colour to highlight raw strings.
+            // It's not exactly a heredoc, but it's the closest!
+            if (!addRun(kBBLMHereDocStringRunKind, runStart, runLen, *callbacks)) return noErr;
+            runStart = iter.Offset();
+        }
+        
+        else if (ch == '"')
         {
             iter--;
             if (!makeCodeRun(iter, runStart, *callbacks)) return noErr;
@@ -493,17 +647,143 @@ OSErr calculateRuns(BBLMParamBlock &params, const BBLMCallbackBlock *callbacks)
             }
         }
 
-        else if (ch == 'f')
+        else if (!wordchr && ch == 'm')
+        {
+            ch = iter.GetNextChar();
+            if (ch == 'o')
+            {
+                ch = iter.GetNextChar();
+                if (ch == 'd')
+                {
+                    ch = iter.GetNextChar();
+                    if (isspace(ch))
+                    {
+                        if (!makeCodeRun(iter, runStart, *callbacks)) return noErr;
+                        runStart = iter.Offset();
+                        runLen = skipWhitespace(iter);
+                        runLen += skipWord(iter);
+                        if (!addRun(moduleColour, runStart, runLen, *callbacks)) return noErr;
+                        runStart = iter.Offset();
+                    }
+                    else if (ch)
+                    {
+                        iter--;
+                    }
+                }
+                else if (ch)
+                {
+                    iter--;
+                }
+            }
+            else if (ch == 'a' && iter.strcmp("cro_rules!", 10) == 0)
+            {
+                iter += 10;
+                ch = iter.GetNextChar();
+                if (isspace(ch))
+                {
+                    if (!makeCodeRun(iter, runStart, *callbacks)) return noErr;
+                    runStart = iter.Offset();
+                    runLen = skipWhitespace(iter);
+                    runLen += skipWord(iter);
+                    if (!addRun(functionColour, runStart, runLen, *callbacks)) return noErr;
+                    runStart = iter.Offset();
+                }
+                else if (ch)
+                {
+                    iter--;
+                }
+            }
+            else if (ch)
+            {
+                iter--;
+            }
+        }
+
+        else if (!wordchr && ch == 'f')
         {
             ch = iter.GetNextChar();
             if (ch == 'n')
             {
-                if (!makeCodeRun(iter, runStart, *callbacks)) return noErr;
+                ch = iter.GetNextChar();
-                runStart = iter.Offset();
+                if (isspace(ch))
-                runLen = skipWhitespace(iter);
+                {
-                runLen += skipWord(iter);
+                    if (!makeCodeRun(iter, runStart, *callbacks)) return noErr;
-                if (!addRun(functionColour, runStart, runLen, *callbacks)) return noErr;
+                    runStart = iter.Offset();
-                runStart = iter.Offset();
+                    runLen = skipWhitespace(iter);
+                    runLen += skipWord(iter);
+                    if (!addRun(functionColour, runStart, runLen, *callbacks)) return noErr;
+                    runStart = iter.Offset();
+                }
+                else if (ch)
+                {
+                    iter--;
+                }
+            }
+            else if (ch)
+            {
+                iter--;
+            }
+        }
+
+        else if (!wordchr && ch == 'u')
+        {
+            ch = iter.GetNextChar();
+            if (ch == 's')
+            {
+                ch = iter.GetNextChar();
+                if (ch == 'e')
+                {
+                    ch = iter.GetNextChar();
+                    if (isspace(ch))
+                    {
+                        skipWhitespace(iter);
+                        if (!makeCodeRun(iter, runStart, *callbacks)) return noErr;
+
+                        runStart = iter.Offset();
+                        runLen = 0;
+                        bool spacey = false;
+
+                        while ((ch = iter.GetNextChar()))
+                        {
+                            if (spacey && isupper(ch))
+                            {
+                                iter--;
+                                if (!addRun(kBBLMFileIncludeRunKind, runStart, runLen, *callbacks)) return noErr;
+
+                                runStart = iter.Offset();
+                                runLen = skipWord(iter);
+                                if (!addRun(identifierColour, runStart, runLen, *callbacks)) return noErr;
+
+                                runStart = iter.Offset();
+                                runLen = 0;
+                            }
+                            else if (ch == ';' || ch == '\n')
+                            {
+                                iter--;
+                                if (!addRun(kBBLMFileIncludeRunKind, runStart, runLen, *callbacks)) return noErr;
+                                break;
+                            }
+                            else
+                            {
+                                spacey = isspace(ch) || ch == ':' || ch == '{';
+                                runLen++;
+                            }
+                        }
+
+                        if (!addRun(kBBLMFileIncludeRunKind, runStart, runLen, *callbacks)) return noErr;
+
+                        runStart = iter.Offset();
+                        runLen = skipWord(iter);
+                    }
+                    else if (ch)
+                    {
+                        iter--;
+                    }
+                }
+                else if (ch)
+                {
+                    iter--;
+                }
             }
             else if (ch)
             {
@@ -544,7 +824,18 @@ OSErr calculateRuns(BBLMParamBlock &params, const BBLMCallbackBlock *callbacks)
             if (!makeCodeRun(iter, runStart, *callbacks)) return noErr;
             runStart = iter.Offset();
             runLen = skipAttribute(iter);
-            if (!addRun(attributeColour, runStart, runLen, *callbacks)) return noErr;
+            if (!addRun(kBBLMPreprocessorRunKind, runStart, runLen, *callbacks)) return noErr;
+            runStart = iter.Offset();
+        }
+        
+        else if (ch == '$')
+        {
+            iter--;
+            if (!makeCodeRun(iter, runStart, *callbacks)) return noErr;
+            runStart = iter.Offset();
+            iter++;
+            runLen = skipWord(iter) + 1;
+            if (runLen > 1 && !addRun(kBBLMVariableRunKind, runStart, runLen, *callbacks)) return noErr;
             runStart = iter.Offset();
         }
 
@@ -558,6 +849,25 @@ OSErr calculateRuns(BBLMParamBlock &params, const BBLMCallbackBlock *callbacks)
             runStart = iter.Offset();
         }
 
+        else if (!wordchr && (ch == '+' || ch == '-'))
+        {
+            ch = iter.GetNextChar();
+            if (isdigit(ch))
+            {
+                iter -= 2;
+                if (!makeCodeRun(iter, runStart, *callbacks)) return noErr;
+                iter++;
+                runStart = iter.Offset() - 1;
+                runLen = skipNumber(iter) + 1;
+                if (!addRun(kBBLMNumberRunKind, runStart, runLen, *callbacks)) return noErr;
+                runStart = iter.Offset();
+            }
+            else if (ch)
+            {
+                iter--;
+            }
+        }
+
         else if (!wordchr && isdigit(ch))
         {
             iter--;
@@ -568,21 +878,22 @@ OSErr calculateRuns(BBLMParamBlock &params, const BBLMCallbackBlock *callbacks)
             runStart = iter.Offset();
         }
 
-        wordchr = isalpha(ch) || isdigit(ch);
+        wordchr = isalpha(ch) || isdigit(ch) || ch == '_';
     }
 
     makeCodeRun(iter, runStart, *callbacks);
     return noErr;
 }
 
-static bool isSpecialKind(NSString* kind)
+/*static bool isSpecialKind(NSString* kind)
 {
     return [kBBLMBlockCommentRunKind isEqualToString:kind]
         || [kBBLMLineCommentRunKind isEqualToString:kind]
         || [identifierColour isEqualToString:kind]
         || [attributeColour isEqualToString:kind]
-        || [lifetimeColour isEqualToString:kind];
+        || [lifetimeColour isEqualToString:kind]
-}
+        || [functionColour isEqualToString:kind];
+}*/
 
 OSErr adjustRange(BBLMParamBlock &params, const BBLMCallbackBlock &callbacks)
 {
@@ -592,7 +903,7 @@ OSErr adjustRange(BBLMParamBlock &params, const BBLMCallbackBlock &callbacks)
     SInt32 length;
     UInt32 index = params.fAdjustRangeParams.fStartIndex;
 
-    while (index > 0 && bblmGetRun(&callbacks, index, language, kind, charPos, length) && isSpecialKind(kind))
+    while (index > 0 && bblmGetRun(&callbacks, index, language, kind, charPos, length)/* && isSpecialKind(kind)*/)
     {
         index--;
     }

smoke.rs

@@ -0,0 +1,120 @@
+// Keywords
+// --------
+
+  abstract, alignof, as, become, box, break, const, continue, crate, do, else, enum,
+  extern, false, final, fn, for, if, impl, in, let, loop, macro, match, mod, move, mut,
+  offsetof, override, priv, pub, pure, ref, return, sizeof, static, self, struct, super,
+  true, trait, type, typeof, unsafe, unsized, use, virtual, where, while, yield.
+
+// Types
+// -----
+
+  - ()
+  - bool, char, str
+  - u8, u16, u32, u64
+  - i8, i16, i32, i64
+  - f32, f64
+  - usize, isize
+
+  - int, uint    // these are no longer valid!
+
+// Comments
+// --------
+
+  // This is a comment
+  /* So is this */
+
+  /// This is a doc comment
+  /** So is this */
+
+// Characters and Lifetimes
+// ------------------------
+
+  'a'        // a character
+  'a         // a lifetime
+  'static    // also a lifetime
+  b'a'       // a byte
+
+  ' '        // this is also a character (space)
+  '\t'       // characters can be escaped
+  '\''       // including the quote character itself!
+
+  "geese"          // a string
+  r##"geese"##     // a raw string. This part should not be highlighted!
+  b"geese"         // a byte string
+  br##"geese"##    // a raw byte string. This part should still not be highlighted
+
+  "there's a \" in this string"  // be sure to handle escapes
+  "\"geese\""                    // including at the edges
+
+  r##"there's a " in this string"##  // the string shouldn't end until ##
+  r##"there's a # in this string"##  // even if there's a # in the middle
+
+  // " // in case the above failed :)
+
+// Decimal Numbers
+// ---------------
+
+  7, 12, 1048576    // are all numbers
+  1_000_000         // is a number with underscore separators
+  _, _1000, 1000_   // beware of sole/leading/trailing underscores
+
+// Hex Numbers
+// -----------
+
+  0xff              // hex numbers start with 0x
+  0xFF              // and can be in lowercase or uppercase
+  0x0123_4567       // and can have separators
+
+// Octal Numbers
+// -------------
+
+  0o77              // octal numbers start with 0o
+  0o12345678909     // and should finish after an 8 or 9
+
+// Binary Numbers
+// --------------
+
+  0b0000_1111       // binary numbers start with 0b
+  0b01234311        // and should finish after a 2 or higher
+
+// Floating-Point Numbers
+// ----------------------
+
+  6.022E-23         // floating point notation
+  5.974e24          // lowercase e
+  7.348e+22         // uppercase E and plus
+
+  let tuple = ((8, 4), 48);
+  let a = tuple.0.0;
+
+// Numeric Suffixes
+// ----------------
+
+  123i32,  123u32   // numbers can have literals
+  123_u32, 0xff_u8
+  0o70_i16, 0b1111_1111_1001_0000_i32
+  0us, 7is
+
+  123.0f64
+  0.1f64
+  0.1f32
+  12E+99_f64
+  2.f64  // Invalid!
+
+// Attributes
+// ----------
+
+  #[attribute]
+  #[derive(This, That, Other)]
+  #![top_level_attribute]
+  #not_an_attribute
+
+  #[macro_use] use this_is_not_an_attribute;
+
+// Macros
+// ------
+
+  macro_rules! parse {
+      ($thing: expr) => { $thing };
+  }