1 module rest.apiv1; 2 3 import vibe.core.log; 4 import vibe.data.json; 5 import vibe.http.client; 6 import vibe.stream.operations : readAllUTF8; 7 8 import rest.iapiv1; 9 import exec.iexecprovider; 10 import contentprovider; 11 12 import std.array : replace; 13 import std.format : format; 14 15 class ApiV1: IApiV1 16 { 17 private IExecProvider execProvider_; 18 private ContentProvider contentProvider_; 19 string githubToken; /// Token for authenticating GitHub requests 20 21 this(IExecProvider execProvider, ContentProvider contentProvider, string githubToken) 22 { 23 this.execProvider_ = execProvider; 24 this.contentProvider_ = contentProvider; 25 this.githubToken = "token " ~ githubToken; 26 } 27 28 /++ 29 Parses the message contained in $(D output) 30 and fills the appropriate errors and warnings 31 arrays. 32 Note: parsing s just done when $(D output.success) 33 is false. 34 +/ 35 private static void parseErrorsAndWarnings(ref RunOutput output) 36 { 37 import std.regex: ctRegex, matchFirst, replaceAll; 38 import std.conv: to; 39 import std..string: lineSplitter; 40 41 static coloring = ctRegex! 42 `\x1B\[[0-9;]*[mGK]`; 43 static ctr = ctRegex! 44 `^[^(]+\(([0-9]+)(,[0-9]+)?\): ([a-zA-Z]+): (.*)$`; 45 46 foreach(line; output.output.replaceAll(coloring, "").lineSplitter) { 47 auto m = line.matchFirst(ctr); 48 if (m.empty) 49 continue; 50 auto lineNumber = to!int(m[1]); 51 string type = m[3]; 52 string message = m[4]; 53 54 switch (type) { 55 case "Warning": 56 case "Deprecation": 57 output.warnings ~= RunOutput.Message(lineNumber, 58 message); 59 break; 60 default: 61 output.errors ~= RunOutput.Message(lineNumber, 62 message); 63 } 64 } 65 } 66 67 // --- Argumentation for the chosen limit --- 68 // The average line length of Phobos std/ is about 30 characters, 69 // while on average files are 1800 lines long. The average file 70 // size is 62391 bytes. 71 // We pick a rather generous limit of 64k (64 * 1024) which ought 72 // to be enough for everybody. 73 enum uint maxSourceCodeLength = 64 * 1024; 74 75 RunOutput run(ApiV1.RunInput input) 76 { 77 if (input.source.length > maxSourceCodeLength) { 78 return RunOutput("ERROR: source code size is above limit of 64k bytes.", false); 79 } 80 81 // sanitize input and remove weird unicode characters 82 input.source = input.source.removeUnicodeSpaces; 83 84 // Be explicit here, in case the exec API and the REST API change 85 IExecProvider.RunInput runInput = { 86 source: input.source, 87 compiler : input.compiler, 88 args: input.args, 89 runtimeArgs: input.runtimeArgs, 90 stdin: input.stdin, 91 color: input.color, 92 }; 93 auto result = execProvider_.compileAndExecute(runInput); 94 auto output = RunOutput(result.output, result.success); 95 parseErrorsAndWarnings(output); 96 return output; 97 } 98 99 FormatOutput format(string source) 100 { 101 // https://github.com/dlang-community/dfmt/blob/master/src/dfmt/config.d 102 import dfmt.config : Config; 103 import dfmt.formatter : format; 104 import std.array : appender; 105 import std.range.primitives; 106 107 FormatOutput output; 108 ubyte[] buffer; 109 buffer = cast(ubyte[]) source; 110 111 Config formatterConfig; 112 formatterConfig.initializeWithDefaults(); 113 auto app = appender!string; 114 format(source, buffer, app, &formatterConfig); 115 output.source = app.data; 116 if (output.source.back == '\n') 117 output.source.popBack; 118 return output; 119 } 120 121 ShortenOutput shorten(string source, string compiler, string args) 122 { 123 import std.format : format; 124 import std.uri : encodeComponent; 125 126 ShortenOutput output; 127 auto url = "https://run.dlang.io?compiler=%s&source=%s".format(compiler, source.encodeComponent); 128 if (args.length > 0) 129 url ~= "&args=" ~ args.encodeComponent; 130 131 auto isURL= "https://is.gd/create.php?format=simple&url=%s".format(url.encodeComponent); 132 output.url = requestHTTP(isURL, (scope req) { 133 req.method = HTTPMethod.POST; 134 }).bodyReader.readAllUTF8.replace("https://is.gd/", "https://run.dlang.io/is/"); 135 output.success = true; 136 return output; 137 } 138 139 GistOutput gist(string source, string compiler, string args) 140 { 141 import std.format : format; 142 import std.uri : encodeComponent; 143 144 GistOutput output; 145 146 auto res = requestHTTP("https://api.github.com/gists", (scope req) { 147 req.headers["Authorization"] = githubToken; 148 Json data = Json.emptyObject; 149 data["description"] = "Code shared from run.dlang.io.%s".format(args ? 150 " Run with '%s'".format(args) : ""); 151 data["public"] = true; 152 Json files = Json.emptyObject; 153 files["main.d"] = Json([ 154 "content": Json(source), 155 ]); 156 data["files"] = files; 157 req.method = HTTPMethod.POST; 158 req.writeJsonBody(data); 159 }); 160 enforceHTTP(res.statusCode / 100 == 2, HTTPStatus.internalServerError, "GitHub API not available."); 161 auto json = res.readJson; 162 auto gistId = json["id"].get!string; 163 logInfo("Gist created: %s", gistId); 164 output.id = gistId; 165 output.htmlUrl = json["html_url"].get!string; 166 167 output.url = "/gist/" ~ gistId; 168 if (compiler != "dmd") { 169 output.url ~= "?compiler=%s".format(compiler); 170 if (args.length > 0) 171 output.url ~= "&args=" ~ args.encodeComponent; 172 } else if (args.length > 0) { 173 output.url ~= "?args=" ~ args.encodeComponent; 174 } 175 176 return output; 177 } 178 179 unittest 180 { 181 string source = `void main() {}`; 182 ApiV1 api = new ApiV1(null, null, null); 183 auto res = api.format(source); 184 assert(res == FormatOutput("void main()\n{\n}", false)); 185 } 186 187 SourceOutput getSource(string _language, string _chapter, string _section) 188 { 189 auto tourData = contentProvider_.getContent(_language, _chapter, _section); 190 if (tourData.content == null) { 191 throw new HTTPStatusException(404, 192 "Couldn't find tour data for chapter '%s', section %d".format(_language, _chapter, _section)); 193 } 194 195 return SourceOutput(tourData.content.sourceCode); 196 } 197 } 198 199 unittest { 200 string run1 = `onlineapp.d(6): Error: found '}' when expecting ';' following statement 201 onlineapp.d(6): Error: found 'EOF' when expecting '}' following compound statement 202 Failed: ["dmd", "-v", "-o-", "onlineapp.d", "-I."]`; 203 204 auto test2 = ApiV1.RunOutput(run1, false); 205 ApiV1.parseErrorsAndWarnings(test2); 206 assert(test2.errors.length == 2); 207 assert(test2.errors[0].line == 6); 208 assert(test2.errors[0].message[0 .. 9] == "found '}'"); 209 assert(test2.errors[1].line == 6); 210 assert(test2.errors[1].message[0 .. 11] == "found 'EOF'"); 211 212 string run2 = `dlang-tour ~master: building configuration "executable"... 213 ../.dub/packages/dyaml-0.5.2/source/dyaml/dumper.d(15,8): Deprecation: module std.stream is deprecated - It will be removed from Phobos in October 2016. If you still need it, go to https://github.com/DigitalMars/undeaD 214 ../.dub/packages/dyaml-0.5.2/source/dyaml/emitter.d(21,8): Deprecation: module std.stream is deprecated - It will be removed from Phobos in October 2016. If you still need it, go to https://github.com/DigitalMars/undeaD 215 ../.dub/packages/dyaml-0.5.2/source/dyaml/representer.d(679,8): Deprecation: module std.stream is deprecated - It will be removed from Phobos in October 2016. If you still need it, go to https://github.com/DigitalMars/undeaD 216 ../.dub/packages/dyaml-0.5.2/source/dyaml/loader.d(171,16): Deprecation: module std.stream is deprecated - It will be removed from Phobos in October 2016. If you still need it, go to https://github.com/DigitalMars/undeaD`; 217 auto test3 = ApiV1.RunOutput(run2, false); 218 ApiV1.parseErrorsAndWarnings(test3); 219 assert(test3.warnings.length == 4); 220 import std.algorithm: all; 221 assert(test3.warnings.all!(x => x.message[0 .. 31] == "module std.stream is deprecated")); 222 assert(test3.warnings[0].line == 15); 223 assert(test3.warnings[1].line == 21); 224 assert(test3.warnings[2].line == 679); 225 assert(test3.warnings[3].line == 171); 226 227 auto run3 = 228 "\u001B[1md.d(1): \u001B[1;31mError: \u001B[mbasic type expected, not {\n" ~ 229 "\u001B[1md.d(1): \u001B[1;31mError: \u001B[mfound '{' when expecting ')'\n" ~ 230 "\u001B[1md.d(1): \u001B[1;31mError: \u001B[msemicolon expected following function declaration\n" ~ 231 "\u001B[1md.d(1): \u001B[1;31mError: \u001B[munrecognized declaration\n"; 232 233 auto test4 = ApiV1.RunOutput(run3, false); 234 ApiV1.parseErrorsAndWarnings(test4); 235 236 assert(test4.errors.length == 4); 237 assert(test4.errors[0] == 238 ApiV1.RunOutput.Message(1, "basic type expected, not {")); 239 assert(test4.errors[1] == 240 ApiV1.RunOutput.Message(1, "found '{' when expecting ')'")); 241 assert(test4.errors[2] == 242 ApiV1.RunOutput.Message(1, "semicolon expected following function declaration")); 243 assert(test4.errors[3] == 244 ApiV1.RunOutput.Message(1, "unrecognized declaration")); 245 } 246 247 // remove weird unicode spaces 248 private string removeUnicodeSpaces(S)(S input) { 249 import std.algorithm.iteration : map; 250 import std.array : array; 251 import std.uni : isSpace; 252 // selective import will trigger false-positive deprecation warning 253 import std.utf; 254 return (cast(dstring) input.map!(c => c.isSpace ? ' ' : c).array).toUTF8; 255 } 256 257 // weird 0xa0 unicode space tokens on android 258 unittest { 259 import std..string : representation; 260 ubyte[] helloWorldNormal = [10, 32, 32, 32, 32, 32, 32, 119, 114, 105]; 261 ubyte[] helloWorldAndroid = [10, 32, 194, 160, 32, 194, 160, 32, 32, 119, 114, 105]; 262 string res = cast(string) helloWorldAndroid; 263 assert(res.removeUnicodeSpaces.representation == helloWorldNormal); 264 }