Browse Source

Added better benchmarks

crazy_things
Ludovic 'Archivist' Lagouardette 1 year ago
parent
commit
f287f2752d
9 changed files with 159 additions and 23 deletions
  1. +1
    -0
      CMakeLists.txt
  2. +2
    -0
      include/UserScript.h
  3. +8
    -0
      priv_include/UserScript/interpreter.h
  4. +41
    -19
      script_exe/main.cpp
  5. +99
    -2
      src/std/crypto.cpp
  6. +4
    -1
      tests/parser_test.cpp
  7. +2
    -0
      tests/scripts/007.results
  8. +2
    -0
      tests/scripts/007.script
  9. +0
    -1
      tests/scripts/testfile.test

+ 1
- 0
CMakeLists.txt View File

@ -52,3 +52,4 @@ add_script_test("Scripting 003: While loops" tests/scrip
add_script_test("Scripting 004: While loops with bad terminator" tests/scripts/004.script tests/scripts/004.results)
add_script_test("Scripting 005: If statements with bad terminator" tests/scripts/005.script tests/scripts/005.results)
add_script_test("Scripting 006: The stack is properly purged" tests/scripts/006.script tests/scripts/006.results)
add_script_test("Scripting 007: Cryptography seems to work" tests/scripts/007.script tests/scripts/007.results)

+ 2
- 0
include/UserScript.h View File

@ -49,6 +49,8 @@ namespace scripting {
virtual std::optional<std::reference_wrapper<script_value>> getValue(const std::string& name) = 0;
virtual bool setValue(const std::string& name, script_value value) = 0;
virtual void registerFunction(std::string name, function fn) = 0;
virtual void clear_variables() = 0;
virtual int32_t op_count() = 0;
virtual script_value resolve(const std::string& name) = 0;
virtual std::variant<script_value, std::vector<script_error>> executeAtOnce(std::string code) = 0;
virtual std::vector<script_error> prepare(std::string code) = 0;

+ 8
- 0
priv_include/UserScript/interpreter.h View File

@ -113,6 +113,14 @@ namespace scripting {
functions.insert_or_assign(name, std::move(fn));
}
void clear_variables() final {
variables.clear();
}
int32_t op_count() final {
return bytecode.size();
}
std::variant<script_value, std::vector<script_error>> executeAtOnce(std::string code) final {
std::vector<script_error> errors;
auto lexed = ast::lex(code, errors);

+ 41
- 19
script_exe/main.cpp View File

@ -133,6 +133,11 @@ void process_bench(std::string target = "./tests/scripts/testfile.test") {
* 2023-07-04 Archivist -> 2618ns - 308ns - 49ns (clang+libstdc++)
* 2023-07-07 Archivist -> 2481ns - 291ns - 46ns (clang+libc++)
* 2023-07-07 Archivist -> 106ns - 12ns - 2ns (clang+march=native+libc++)
* ?????????? Archivist: These results above are weird, the benchmark was updated in case something strange went on
* 2023-07-18 Archivist -> 3446ns - 405ns - 65ns (gcc+libstdc++)
* ?????????? Archivist: Corrected some extra weirdness
* 2023-07-18 Archivist -> 9952µs - 234ns - 30ns (gcc+libstdc++)
*
*/
engine->registerFunction("print", std::make_unique<print>(std::cout));
std::ifstream src_str(target);
@ -142,10 +147,12 @@ void process_bench(std::string target = "./tests/scripts/testfile.test") {
decltype(std::chrono::high_resolution_clock::now()-std::chrono::high_resolution_clock::now()) per_exec{}, per_step{}, per_op{};
for(int runs = 0; runs < 5000; runs++) {
constexpr size_t runs_count = 500;
for(int runs = 0; runs < runs_count; runs++) {
auto res = engine->prepare(code.str());
engine->clear_variables();
auto begin = std::chrono::high_resolution_clock::now();
while (not engine->getValue("exit_ctr").has_value()) {
engine->stepOnce();
@ -156,13 +163,16 @@ void process_bench(std::string target = "./tests/scripts/testfile.test") {
per_step += (end - begin);
per_op += (end - begin);
}
per_exec /= 5000;
auto executed_ops = (runs_count * engine->op_count() * 5000 /* The code loops 5000 times */);
per_exec /= runs_count;
per_step /= steps;
per_op = per_op / 5000 / 53;
per_op = per_op / n">executed_ops;
std::cout << "time per exec = " << std::chrono::duration_cast<std::chrono::nanoseconds>(per_exec).count() << "ns\n";
std::cout << "time per step = " << std::chrono::duration_cast<std::chrono::nanoseconds>(per_step).count() << "ns\n";
std::cout << "time per avg op = " << std::chrono::duration_cast<std::chrono::nanoseconds>(per_op).count() << "ns\n";
std::cout << "time per exec (" << runs_count << ") = " << std::chrono::duration_cast<std::chrono::microseconds>(per_exec).count() << "µs\n";
std::cout << "time per step (" << steps << ", around " << (double)executed_ops / steps << "op/s) = " << std::chrono::duration_cast<std::chrono::nanoseconds>(per_step).count() << "ns\n";
std::cout << "time per avg op (" << executed_ops << " by groups of " << engine->op_count() << ") = " << std::chrono::duration_cast<std::chrono::nanoseconds>(per_op).count() << "ns\n";
}
void compile_bench(std::string target = "./tests/scripts/testfile.test") {
@ -176,26 +186,34 @@ void compile_bench(std::string target = "./tests/scripts/testfile.test") {
* Same as above but for compilation times
*
* 2023-07-04 Archivist -> 386µs
* 2023-07-18 Archivist -> 166µs
* 2023-07-18 Archivist -> 156µs
*/
engine->registerFunction("print", std::make_unique<print>(std::cout));
std::ifstream src_str("./tests/scripts/testfile.test");
std::stringstream code;
code << src_str.rdbuf();
auto begin = std::chrono::high_resolution_clock::now();
constexpr size_t daruns_count = 100;
[&]() __attribute__((optimize("O0"))) {
auto res = engine->prepare(code.str());
res = engine->prepare(code.str());
res = engine->prepare(code.str());
res = engine->prepare(code.str());
res = engine->prepare(code.str());
}();
auto begin = std::chrono::high_resolution_clock::now();
for(size_t i = 0; i < daruns_count; ++i) {
[&]() __attribute__((optimize("O0"))) {
auto res = engine->prepare(code.str());
res = engine->prepare(code.str());
res = engine->prepare(code.str());
res = engine->prepare(code.str());
res = engine->prepare(code.str());
res = engine->prepare(code.str());
res = engine->prepare(code.str());
res = engine->prepare(code.str());
res = engine->prepare(code.str());
res = engine->prepare(code.str());
}();
}
auto end = std::chrono::high_resolution_clock::now();
auto per_exec = (end - begin)/5;
auto per_exec = (end - begin) / 10 / daruns_count;
std::cout << "time per exec = " << std::chrono::duration_cast<std::chrono::microseconds>(per_exec).count() << "µs\n";
}
@ -206,7 +224,11 @@ void compare(std::string target, std::string expect) {
engine->registerFunction("exit", std::make_unique<terminate>());
engine->registerFunction("set", std::make_unique<set>());
engine = scripting::register_array_lib(std::move(engine), true, 4096);
constexpr size_t array_limit = 4096;
constexpr size_t string_limit = 4096;
engine = scripting::register_array_lib(std::move(engine), true, array_limit);
engine = scripting::register_crypto_lib(std::move(engine), array_limit, string_limit);
std::stringstream str;
std::string_view filename_source = target;

+ 99
- 2
src/std/crypto.cpp View File

@ -186,7 +186,7 @@ namespace scripting {
struct fn_encode_n final : public scripting::function_impl {
int32_t string_size_limit;
explicit fn_encode_n(int32_t _string_size_limit)
: string_size_limit(_string_size_limit) {}
: string_size_limit(_string_size_limit) {}
std::optional<script_value> apply(UserScript* self, std::vector<argument> n, std::optional<script_error>& error) final {
using verifier = Verify<
@ -279,10 +279,107 @@ namespace scripting {
~fn_encode_n() final = default;
};
struct fn_decode_n final : public scripting::function_impl {
int32_t string_size_limit;
explicit fn_decode_n(int32_t _string_size_limit)
: string_size_limit(_string_size_limit) {}
std::optional<script_value> apply(UserScript* self, std::vector<argument> n, std::optional<script_error>& error) final {
using verifier = Verify<
details::SizeEquals<4>,
details::TypeVerifier<0, int32_t>,
details::TypeVerifier<1, int32_t>,
details::TypeVerifier<2, int32_t>,
details::OctetArrayVerifier<3>
>;
if(not verifier{}.verify(self, n)) {
error = script_error{
.message = "/decode_n takes exactly 3 arguments of type integer and an argument of type array"
};
return std::nullopt;
}
auto& arg = n.front();
script_value target;
if(std::holds_alternative<scripting::script_value>(arg)) {
target = std::get<scripting::script_value>(arg);
} else {
target = self->resolve(std::get<scripting::script_variable>(arg).name);
}
bad_cryptography::encode_n_parameters params;
{
auto& arg = n[3];
script_value target;
if (std::holds_alternative<scripting::script_value>(arg)) {
target = std::get<scripting::script_value>(arg);
} else {
target = self->resolve(std::get<scripting::script_variable>(arg).name);
}
params.spin = std::get<int32_t>(target);
}
{
auto& arg = n[2];
script_value target;
if (std::holds_alternative<scripting::script_value>(arg)) {
target = std::get<scripting::script_value>(arg);
} else {
target = self->resolve(std::get<scripting::script_variable>(arg).name);
}
params.clash = std::get<int32_t>(target);
}
{
auto& arg = n[1];
script_value target;
if (std::holds_alternative<scripting::script_value>(arg)) {
target = std::get<scripting::script_value>(arg);
} else {
target = self->resolve(std::get<scripting::script_variable>(arg).name);
}
params.maim = std::get<int32_t>(target);
}
auto ary = std::get<array>(target);
if(ary.value.size() > string_size_limit) {
error = script_error{
.message = "/decode_n: array is too bit to fit string type"
};
return std::nullopt;
}
std::transform(ary.value.begin(), ary.value.end(), ary.value.begin(), [&](const script_value& value) {
return std::visit(
wizardry::overloaded{
[&](int32_t v) -> script_value { return script_value{bad_cryptography::decode_n_1(v, params)};},
[](auto v) -> script_value { return null{}; }
},
value
);
});
return ary;
}
~fn_decode_n() final = default;
};
interpreter register_crypto_lib(interpreter target, int32_t array_size_limit, int32_t string_size_limit) {
target->registerFunction("string_to_binary", std::make_unique<fn_string_to_binary>(array_size_limit));
target->registerFunction("binary_to_string", std::make_unique<fn_binary_to_string>(string_size_limit));
target->registerFunction("encode_n", std::make_unique<fn_encode_n>(string_size_limit));
target->registerFunction("decode_n", std::make_unique<fn_decode_n>(string_size_limit));
return std::move(target);
}
}
}

+ 4
- 1
tests/parser_test.cpp View File

@ -84,6 +84,8 @@ constexpr auto runner = [](){
"../tests/scripts/003.script",
"../tests/scripts/004.script",
"../tests/scripts/005.script",
"../tests/scripts/006.script",
"../tests/scripts/007.script",
};
auto seed = seed_template == -1 ? std::random_device{}() : seed_template;
@ -122,7 +124,7 @@ constexpr auto runner = [](){
size_t count = 0;
size_t error_cnt = 0;
size_t success_cnt = 0;
constexpr size_t max_count = 5000000;
constexpr size_t max_count = 7000000;
auto begin = std::chrono::high_resolution_clock::now();
while(count < max_count) {
@ -157,6 +159,7 @@ constexpr auto runner = [](){
};
TEST_CASE("Try to crash the parser (known seeds)") {
runner<4138740281>();
runner<1547293717>();
runner<1759257947>();
runner<2909912711>();

+ 2
- 0
tests/scripts/007.results View File

@ -0,0 +1,2 @@
Hello
Potato

+ 2
- 0
tests/scripts/007.script View File

@ -0,0 +1,2 @@
/print (/binary_to_string (/decode_n 1 2 3 (/encode_n 1 2 3 (/string_to_binary "Hello\n"))))
/print (/binary_to_string (/decode_n 10 20 30 (/encode_n 10 20 30 (/string_to_binary "Potato\n"))))

+ 0
- 1
tests/scripts/testfile.test View File

@ -1,6 +1,5 @@
if(counter == (/null))
/set counter 0
/print "Init...\n"
endif
/bigDoNothing 17 12 36*78
if(counter % 2 == 1)

Loading…
Cancel
Save