// This file is part of VPL for Moodle - http://vpl.dis.ulpgc.es/
//
// VPL for Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// VPL for Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with VPL for Moodle.  If not, see <http://www.gnu.org/licenses/>.

/**
 * Example of a tokenizer rules declaration
 *
 *  Tokenizer needs to know what is the lexic that defines
 *  a programming language. This JSON file would contain all
 *  the information that tokenizer uses, such as the states,
 *  programming language extension, and so on.
 *
 *  The example is just a brief demostration of the tools that
 *  tokenizer allows in order to define a set of rules. For more
 *  information about tokenizer rules, please see <...>.
 *
 *  This syntax is based on Ace Editor tokenizer rules files. To
 *  know more about them, go to <https://ace.c9.io/>
 */
{
    /**
     * As it is shown here, this files could have comments based
     * on C-style, so you can include inline or multiple-line
     * comments, as it is define in C programming language.
     *
     * Talking about the performance, comments would be discarded
     * during the creation of the tokenizer, so in some cases on
     * which files have a lot of lines, the execution could be slower
     * than expected. However, this are special cases, and you should
     * not worry about them.
     */

    /**
     * Before talking about the content of a tokenizer rules file,
     * it is important to know some basic requirements:
     *
     *  - tokenizer rules must be defined at a JSON file
     *  - The name of the file must end with "_tokenizer_rules.json"
     *  - Some options are secundary, so default values would be used if they're not defined
     */

    // OPTIONAL
    // This option is not required, but could be useful since
    // would define a name for the tokenizer. The tokenizer's
    // name would be "default" whether "name" option is not
    // defined at tokenizer rules file.
    "name": "example-tokenizer",

    // OPTIONAL
    // Each programming language has an extension, and tokenizer would
    // check if passed file ends with that value whether this option
    // has been defined. However, if you want to disable extensions,
    // do not set this option, or define it with "no-ext" value.
    //
    // Since programming languages could be defined with more than
    // one extension, you can define this option with an array of strings
    "extension": "no-ext",   // example: ".c" or [".c",".h"]

    // OPTIONAL
    // Tokenizer would first check if rules are correctly declared.
    // This may cause a slower execution whether there are so many
    // tokenizer rules, so you could skip this step specifying this
    // option. If check_rules has not been declared, its value would be true
    "check_rules": true,

    // OPTIONAL
    // You can inherit the rules of another programming language,
    // so its rules would be applied first. You can omit this
    // property by simply insert an empty value or not declaring it.
    // It is important to notice that file would be searched from
    // current file's path, and extension must not be defined
    "inherit_rules": "text_tokenizer_rules",

    // OPTIONAL
    // Since tokenizer only allows some names for tokens, it is
    // possible to define new tokens or set a different type
    // for an existed one. You can check what types of tokens are
    // at \mod_vpl\tokenizer\token_type. However, it is possible
    // to unset the type of a token by setting its type to "",
    // or set the value of a token using an existed token.
    "override_tokens": {
        "constant": "constant.numeric",
        "string.start": "vpl_literal",
        "string.end": "vpl_literal"
    },

    // OPTIONAL
    // Using this advanced option, it is possible to override the
    // maximum number of tokens allowed during the parse phase.
    // Values must be positive integer numbers.
    "max_token_count": 2000,

    // REQUIRED
    // List of states that tokenizer uses during its execution.
    // Each state must have a named and an array of rules.
    //
    // Tokenizer would always traverse down the list of a the
    // first state defined, and when a regex matches, it will
    // continue to the next state or the one defined at "next".
    "states": {
        // Start state
        //
        //  Initial state on which execution begins. A tokenizer
        //  rules file would be considered as invalid wether this
        //  state has not been defined locally or at inheritance.
        "start": [
            // Definition of a rule, which always contain a set
            // of options. This version only admits "token", "regex",
            // "next", and "default_token", but in future releases
            // "push" and "pop" options would be included.
            //
            // It is important to notice that some options must be
            // defined together, and others need to be alone. For
            // example, "token" and "regex" must be together, but
            // "default_token" should not be with any other option.
            // However, "next" is a special option, since could be
            // defined with "token" and "regex" or alone.
            {
                // "Token" defines the identifier of a rule, and must follow
                // naming conventions of TextMate (https://macromates.com/manual/en/language_grammars).
                // However, there are some valid tokens that are not standard, and would be docummented later.
                "token": "comment", // type: string | array[string] if groups are defined at "regex"

                // "Regex" specifies a regex expression which must match.
                // It is important to notice that you can define groups with "(" and ")"
                // at the expession, but the number of them must match whith the number of tokens at "token".
                "regex": "\\/\\/", // type: string

                // "Next" is an option used to set the state that tokenizer must go
                // whether current input matches the "regex" expression. If "next"
                // is not included, tokenizer would not change current state.
                "next": "single_line_comment" // type: string
            },
            {
                "token": "storage.type",
                "regex": "function\\s*\\(\\w+\\)",
                "next": "declr_function"
            },
            {
                // "Default Token" is an option which must be define alone and should
                // be included at the last rule of the state. It is used to define the
                // token that tokenizer would have whether any of the previous rules match
                "default_token": "comment"  // type: string
            }
        ],
        // Example of a state different than start
        //
        //  Since tokenizerer acts as a state machine, a new
        //  state won't be checked unless one of the regex's
        //  blocks of current state has a next field and matches.
        //  In other words, tokenizerer would check constanly the
        //  actual state until a next token has been actived
        //
        "single_line_comment": [
            {
                "token": "comment",
                "regex": "\\$",
                "next": "single_line_comment"
            },
            {
                "token": "comment",
                "regex": "$",
                "next": "start"
            }
        ],
        "declr_function": [
            {
                "token": [
                    "storage.type",
                    "entity.name",
                    "entity.name"
                ],
                "regex": "(function)(\\s+)([a-zA-Z_][a-zA-Z0-9_]*\\b)",
                "next": "start"
            }
        ]
    }
}