Skip to content

Commit 181ca5d

Browse files
EduardoFernandes1410EduardoFernandes1410
authored
Merge pull request #17 from EOLangVSCode/development
Build improvements + semantic highlighting
2 parents 71d4ed3 + e754a52 commit 181ca5d

File tree

5 files changed

+310
-66
lines changed

5 files changed

+310
-66
lines changed

‎.github/workflows/build.yml‎

Lines changed: 46 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,11 @@ on:
44
branches:
55
- master
66
- development
7+
pull_request:
8+
types: [opened, reopened, edited, synchronize]
9+
branches:
10+
- master
11+
- development
712

813
jobs:
914
build:
@@ -36,16 +41,16 @@ jobs:
3641
run: |
3742
npm i
3843
39-
- name: ESLinter
40-
run: |
41-
npm run lint
42-
4344
- name: SonarCloud Scan
4445
uses: SonarSource/sonarcloud-github-action@master
4546
env:
46-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
47+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
4748
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
4849

50+
- name: ESLinter
51+
run: |
52+
npm run lint
53+
4954
- name: Download and Build Grammar
5055
run: |
5156
cd server
@@ -56,4 +61,39 @@ jobs:
5661
- name: Pre-publish routine
5762
run: |
5863
npm run convert-yaml
59-
npm run vscode:prepublish
64+
npm run vscode:prepublish
65+
66+
# Reference: https://github.com/bahmutov/eleventy-example/blob/main/.github/workflows/ci.yml#L27
67+
68+
- name: Staging Checks ✅
69+
if: ${{ success() }}
70+
# set the merge commit status check
71+
# using GitHub REST API
72+
# see https://docs.github.com/en/rest/reference/repos#create-a-commit-status
73+
run: |
74+
curl --request POST \
75+
--url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.sha }} \
76+
--header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
77+
--header 'content-type: application/json' \
78+
--data '{
79+
"context": "Staging Checks",
80+
"state": "success",
81+
"description": "Staging checks passed",
82+
"target_url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
83+
}'
84+
- name: Staging Checks 🚨
85+
if: ${{ failure() }}
86+
# set the merge commit status check
87+
# using GitHub REST API
88+
# see https://docs.github.com/en/rest/reference/repos#create-a-commit-status
89+
run: |
90+
curl --request POST \
91+
--url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.sha }} \
92+
--header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
93+
--header 'content-type: application/json' \
94+
--data '{
95+
"context": "Staging Checks",
96+
"state": "failure",
97+
"description": "Staging checks failed",
98+
"target_url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
99+
}'

‎server/src/capabilities.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import { ClientCapabilities } from 'vscode-languageserver';
2+
3+
export class Capabilities
4+
{
5+
hasConfigurationCapability: boolean;
6+
hasWorkspaceFolderCapability: boolean;
7+
hasDiagnosticRelatedInformationCapability: boolean;
8+
hasDocumentSemanticTokensCapability: boolean;
9+
10+
constructor () {
11+
this.hasConfigurationCapability = false;
12+
this.hasWorkspaceFolderCapability = false;
13+
this.hasDiagnosticRelatedInformationCapability = false;
14+
this.hasDocumentSemanticTokensCapability = false;
15+
}
16+
17+
initialize (capabilities: ClientCapabilities) {
18+
// Does the client support the `workspace/configuration` request?
19+
// If not, we will fall back using global settings
20+
this.hasConfigurationCapability = !!(capabilities.workspace?.configuration);
21+
this.hasWorkspaceFolderCapability = !!(capabilities.workspace?.workspaceFolders);
22+
this.hasDiagnosticRelatedInformationCapability = !!(capabilities.textDocument?.publishDiagnostics?.relatedInformation);
23+
this.hasDocumentSemanticTokensCapability = !!(capabilities.textDocument?.semanticTokens);
24+
}
25+
}

‎server/src/parser.ts‎

Lines changed: 40 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import * as path from 'path';
33
import {
44
CharStreams,
55
CommonTokenStream,
6-
Token,
6+
Token as AntlrToken,
77
CodePointCharStream,
88
ANTLRErrorListener,
99
Recognizer,
@@ -13,7 +13,6 @@ import {
1313
import { ProgramLexer } from './parser/ProgramLexer';
1414
import { ProgramParser } from './parser/ProgramParser';
1515

16-
1716
class Processor {
1817
inputStream: CodePointCharStream;
1918
lexer: ProgramLexer;
@@ -40,48 +39,55 @@ class Error {
4039
}
4140
}
4241

43-
class ErrorListener implements ANTLRErrorListener<Token> {
42+
class ErrorListener implements ANTLRErrorListener<AntlrToken> {
4443
errorList: Error[] = [];
4544

46-
syntaxError(recognizer: Recognizer<Token, any>, offendingSymbol: Token | undefined, line: number, charPositionInLine: number, msg: string, e: RecognitionException | undefined) {
45+
syntaxError(recognizer: Recognizer<AntlrToken, any>, offendingSymbol: AntlrToken | undefined, line: number, charPositionInLine: number, msg: string, e: RecognitionException | undefined) {
4746
this.errorList.push(new Error(line, charPositionInLine, msg));
4847
}
4948
}
5049

51-
function getTokensNames(path: string): Map<string, string> {
52-
const type2name = new Map<string, string>();
53-
const text = fs.readFileSync(path, {encoding: 'utf-8'});
54-
const tokens = text.split("\n");
55-
tokens.forEach((element: any) => {
56-
if(element[0] != '\'' && element.length > 1) {
57-
const pair = element.split("=");
58-
type2name.set(pair[1], pair[0]);
59-
}
60-
});
61-
return type2name;
50+
let tokenTypes: Set<string> | undefined = undefined;
51+
let tokenNumToString: Map<number, string> | undefined = undefined;
52+
53+
function buildTokenSetAndMap() {
54+
if (!tokenTypes || !tokenNumToString) {
55+
tokenTypes = new Set<string>();
56+
tokenNumToString = new Map<number, string>();
57+
const tokensPath = path.join(__dirname, "../resources/ProgramLexer.tokens");
58+
try {
59+
const text = fs.readFileSync(tokensPath, {encoding: 'utf-8'});
60+
text.split('\n').forEach(elem => {
61+
if (elem[0] != "'") {
62+
const pair = elem.split('=');
63+
tokenTypes!.add(pair[0]);
64+
tokenNumToString!.set(Number(pair[1]), pair[0]);
65+
}
66+
});
67+
} catch (e) {
68+
console.log(e);
69+
}
70+
}
6271
}
6372

64-
export function getTokens(input: string): {name: string, start: number, stop: number}[] {
65-
const type2name = getTokensNames(path.join(__dirname, "../resources/ProgramLexer.tokens"));
66-
const processor = new Processor(input);
67-
68-
const tokenList: any[] = [];
69-
processor.tokenStream.fill();
73+
/**
74+
* Antlr lexer returns token types as numbers
75+
* This converts a type number into textual token type like "META"
76+
*/
77+
export function antlrTypeNumToString(num: number): string {
78+
buildTokenSetAndMap();
79+
return tokenNumToString!.get(num)!;
80+
}
7081

71-
const tokens = processor.tokenStream.getTokens();
82+
export function getTokenTypes(): Set<string> {
83+
buildTokenSetAndMap();
84+
return tokenTypes!;
85+
}
7286

73-
tokens.forEach((element: Token) => {
74-
tokenList.push({
75-
"length": element.stopIndex - element.startIndex + 1,
76-
"name": type2name.get(String(element.type)),
77-
"line": element.line,
78-
"column": element.charPositionInLine,
79-
"start": element.startIndex,
80-
"stop": element.stopIndex
81-
});
82-
});
83-
84-
return tokenList;
87+
export function tokenize(input: string): AntlrToken[] {
88+
const processor = new Processor(input);
89+
processor.tokenStream.fill();
90+
return processor.tokenStream.getTokens();
8591
}
8692

8793
export function getParserErrors(input: string): Error[] {

‎server/src/semantics.ts‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
import
2+
{
3+
SemanticTokensBuilder,
4+
SemanticTokensLegend,
5+
SemanticTokensClientCapabilities
6+
} from 'vscode-languageserver';
7+
import { TextDocument } from 'vscode-languageserver-textdocument';
8+
import { antlrTypeNumToString, tokenize, getTokenTypes } from './parser';
9+
10+
export type VSCodeToken = {
11+
line: number
12+
start: number
13+
length: number
14+
tokenType: number
15+
tokenModifier: number
16+
}
17+
18+
export class SemanticTokensProvider {
19+
legend: SemanticTokensLegend;
20+
/**
21+
* Keep a separate semantic token builder per file
22+
* Each builder keeps track of
23+
*/
24+
tokenBuilders: Map<string, SemanticTokensBuilder> = new Map();
25+
/**
26+
* A map from EO's g4 grammar token types into
27+
* token types supported by VS Code
28+
*/
29+
tokenTypeMap: Map<string, string> = new Map;
30+
31+
constructor(capability: SemanticTokensClientCapabilities) {
32+
this.setMap(); // must run before computing legend!
33+
this.legend = this.computeLegend(capability);
34+
}
35+
36+
setMap() {
37+
/*
38+
Needs update with a proper, fine-tuned mapping
39+
List of VS Code's token types:
40+
[
41+
'namespace', 'type',
42+
'class', 'enum',
43+
'interface', 'struct',
44+
'typeParameter', 'parameter',
45+
'variable', 'property',
46+
'enumMember', 'event',
47+
'function', 'method',
48+
'macro', 'keyword',
49+
'modifier', 'comment',
50+
'string', 'number',
51+
'regexp', 'operator'
52+
]
53+
*/
54+
this.tokenTypeMap.set('COMMENT', 'comment');
55+
this.tokenTypeMap.set('META', 'macro');
56+
this.tokenTypeMap.set('ROOT', 'keyword');
57+
this.tokenTypeMap.set('HOME', 'keyword');
58+
this.tokenTypeMap.set('STAR', 'operator');
59+
this.tokenTypeMap.set('DOTS', 'operator');
60+
this.tokenTypeMap.set('CONST', 'keyword');
61+
this.tokenTypeMap.set('SLASH', 'operator');
62+
this.tokenTypeMap.set('COLON', 'operator');
63+
this.tokenTypeMap.set('COPY', 'class');
64+
this.tokenTypeMap.set('ARROW', 'method');
65+
this.tokenTypeMap.set('VERTEX', 'class');
66+
this.tokenTypeMap.set('SIGMA', 'method');
67+
this.tokenTypeMap.set('XI', 'method');
68+
this.tokenTypeMap.set('PLUS', 'operator');
69+
this.tokenTypeMap.set('MINUS', 'operator');
70+
this.tokenTypeMap.set('QUESTION', 'operator');
71+
this.tokenTypeMap.set('AT', 'method');
72+
this.tokenTypeMap.set('RHO', 'method');
73+
this.tokenTypeMap.set('HASH', 'keyword');
74+
this.tokenTypeMap.set('BYTES', 'number');
75+
this.tokenTypeMap.set('BOOL', 'variable');
76+
this.tokenTypeMap.set('STRING', 'string');
77+
this.tokenTypeMap.set('INT', 'number');
78+
this.tokenTypeMap.set('FLOAT', 'number');
79+
this.tokenTypeMap.set('HEX', 'number');
80+
this.tokenTypeMap.set('NAME', 'variable');
81+
this.tokenTypeMap.set('TEXT', 'string');
82+
}
83+
84+
computeLegend(capability: SemanticTokensClientCapabilities): SemanticTokensLegend {
85+
const clientTokenTypes = new Set<string>(capability.tokenTypes);
86+
87+
const tokenTypes: string[] = [];
88+
getTokenTypes().forEach(el => {
89+
const type = this.tokenTypeMap.get(el) || '';
90+
if (clientTokenTypes.has(type)) {
91+
tokenTypes.push(type);
92+
}
93+
});
94+
95+
return { tokenTypes: tokenTypes, tokenModifiers: [] };
96+
}
97+
98+
tokenize(document: TextDocument) {
99+
const tokens: VSCodeToken[] = [];
100+
const antlrTokens = tokenize(document.getText());
101+
antlrTokens.forEach(tk => {
102+
const vscodeTokenType = this.tokenTypeMap.get(antlrTypeNumToString(tk.type));
103+
const legendNum = vscodeTokenType ? this.legend.tokenTypes.indexOf(vscodeTokenType) : -1;
104+
tokens.push({
105+
line: tk.line - 1,
106+
start: tk.charPositionInLine,
107+
length: tk.stopIndex - tk.startIndex + 1,
108+
tokenType: legendNum,
109+
tokenModifier: 0,
110+
});
111+
});
112+
return tokens;
113+
}
114+
115+
getTokenBuilder(document: TextDocument): SemanticTokensBuilder {
116+
let result = this.tokenBuilders.get(document.uri);
117+
if (!result) {
118+
result = new SemanticTokensBuilder();
119+
this.tokenBuilders.set(document.uri, result);
120+
}
121+
122+
return result;
123+
}
124+
125+
provideSemanticTokens(document: TextDocument) {
126+
const builder = this.getTokenBuilder(document);
127+
this.tokenize(document).forEach((token) =>
128+
{
129+
builder.push(
130+
token.line,
131+
token.start,
132+
token.length,
133+
token.tokenType,
134+
token.tokenModifier
135+
);
136+
});
137+
return builder.build();
138+
}
139+
140+
provideDeltas(document: TextDocument, resultsId: string) {
141+
const builder = this.getTokenBuilder(document);
142+
builder.previousResult(resultsId);
143+
this.tokenize(document).forEach((token) =>
144+
{
145+
builder.push(
146+
token.line,
147+
token.start,
148+
token.length,
149+
token.tokenType,
150+
token.tokenModifier
151+
);
152+
});
153+
return builder.buildEdits();
154+
}
155+
}

0 commit comments

Comments
 (0)