Commit 9e36478e authored by Thomas Ives's avatar Thomas Ives
Browse files

Merge branch 'update_telemetry_tests' into 'main' 

catch2_telemetry.cpp: add tests for telemetry configuration

Admin and device proxy commands are tested that they correctly update the telemetry interface.

Topics are tested to accept 'subscribed to' spans and reject otherwise.

telemetry properties are tested that they override telemetry environment variables.

Closes #1569 .

See merge request !1533
parents b9dfe3f7 cce53ff9

tests/catch2_telemetry.cpp

+575 −1
Original line number Diff line number Diff line
@@ -22,7 +22,8 @@ class TelemetryDS : public Base 


    void read_attribute(Tango::Attribute &att)

    {

        auto span = TANGO_TELEMETRY_SPAN(TELEMETRY_CURRENT_FUNCTION, {{"myKey", "myValue"}});

        auto span =

            TANGO_TELEMETRY_SPAN(TELEMETRY_CURRENT_FUNCTION, {{"myKey", "myValue"}, {"tango.telemetry.topic", "user"}});

        auto scope = TANGO_TELEMETRY_SCOPE(span);



        DEBUG_STREAM << "read_attribute DEBUG_STREAM" << std::endl;
@@ -264,6 +265,579 @@ SCENARIO("Telemetry traces/logs can be turned off") 
    }

}



SCENARIO("Telemetry admin commands: telemetry on/off")

{

    int idlver = GENERATE(TangoTest::idlversion(6));



    GIVEN("a device proxy to a simple IDLv" << idlver << " device and a connection to the admin device")

    {

        std::vector<std::string> env{"TANGO_TELEMETRY_ENABLE=off", "TANGO_TELEMETRY_TYPES=tracing,logging"};

        TangoTest::Context ctx{"telemetry", "TelemetryDS", idlver, env};



        auto device = ctx.get_proxy();



        auto admin = std::make_unique<Tango::DeviceProxy>(device->adm_name());

        REQUIRE(admin != nullptr);



        const std::string devname = "TestServer/tests/1";



        Tango::DeviceData ddin, ddout;

        std::vector<std::string> instr, outstr;

        std::vector<Tango::DevLong> inlong, outlong;



        using namespace TangoTest::Matchers;

        using namespace Catch::Matchers;



        instr = {devname};

        ddin << instr;

        WHEN("we query telemetry status")

        {

            // Check telemetry off

            ddout = admin->command_inout("IsTelemetryEnabled", ddin);

            ddout.extract(outlong, outstr);

            THEN("we find that telemetry is off")

            {

                CHECK_THAT(outlong, RangeEquals(std::vector<long>{0}));

                CHECK_THAT(outstr, RangeEquals(std::vector<std::string>{devname}));

            }

        }



        WHEN("we turn telemetry on and we query telemetry status")

        {

            // Turn telemetry on and check

            ddout = admin->command_inout("StartTelemetry", ddin);

            ddout.reset_exceptions(Tango::DeviceData::isempty_flag);

            CHECK(ddout.is_empty());



            ddout = admin->command_inout("IsTelemetryEnabled", ddin);

            ddout.extract(outlong, outstr);



            THEN("we find that telemetry is enabled")

            {

                CHECK_THAT(outlong, RangeEquals(std::vector<long>{1}));

                CHECK_THAT(outstr, RangeEquals(std::vector<std::string>{devname}));

            }

        }



        WHEN("we turn telemetry on and then off and query telemetry status")

        {

            // Turn telemetry on and check

            ddout = admin->command_inout("StartTelemetry", ddin);

            ddout.reset_exceptions(Tango::DeviceData::isempty_flag);

            CHECK(ddout.is_empty());



            // Turn telemetry off and check

            ddout = admin->command_inout("StopTelemetry", ddin);

            ddout.reset_exceptions(Tango::DeviceData::isempty_flag);

            CHECK(ddout.is_empty());



            ddout = admin->command_inout("IsTelemetryEnabled", ddin);

            ddout.extract(outlong, outstr);

            THEN("we find that telemetry is off")

            {

                CHECK_THAT(outlong, RangeEquals(std::vector<long>{0}));

                CHECK_THAT(outstr, RangeEquals(std::vector<std::string>{devname}));

            }

        }

    }

}



SCENARIO("Telemetry admin commands: logging/tracing status")

{

    int idlver = GENERATE(TangoTest::idlversion(6));



    GIVEN("a device proxy to a simple IDLv" << idlver << " device and a connection to the admin device")

    {

        std::vector<std::string> env{"TANGO_TELEMETRY_ENABLE=off", "TANGO_TELEMETRY_TYPES=tracing,logging"};

        TangoTest::Context ctx{"telemetry", "TelemetryDS", idlver, env};



        auto device = ctx.get_proxy();



        auto admin = std::make_unique<Tango::DeviceProxy>(device->adm_name());

        REQUIRE(admin != nullptr);



        const std::string devname = "TestServer/tests/1";



        Tango::DeviceData ddin, ddout;

        std::vector<std::string> instr, outstr;

        std::vector<Tango::DevLong> inlong, outlong;



        using namespace TangoTest::Matchers;

        using namespace Catch::Matchers;



        std::string telemetry_type = GENERATE("Tracing", "Logging");

        std::string get_command = "GetTelemetry" + telemetry_type;

        std::string set_command = "SetTelemetry" + telemetry_type;



        instr = {devname};



        WHEN("we query " + telemetry_type + " status before turning off")

        {

            // Check type on

            ddin << instr;

            ddout = admin->command_inout(get_command, ddin);

            ddout.extract(outlong, outstr);



            THEN("we find that " + telemetry_type + " is on")

            {

                CHECK_THAT(outlong, RangeEquals(std::vector<long>{1}));

                CHECK_THAT(outstr, RangeEquals(std::vector<std::string>{devname}));

            }

        }



        WHEN("we turn " + telemetry_type + " off and query status")

        {

            // Turn type off and check

            inlong = {0};

            ddin.insert(inlong, instr);

            ddout = admin->command_inout(set_command, ddin);

            ddout.reset_exceptions(Tango::DeviceData::isempty_flag);

            REQUIRE(ddout.is_empty());



            ddin << instr;

            ddout = admin->command_inout(get_command, ddin);

            ddout.extract(outlong, outstr);

            THEN("we find that " + telemetry_type + " is off")

            {

                CHECK_THAT(outlong, RangeEquals(std::vector<long>{0}));

                CHECK_THAT(outstr, RangeEquals(std::vector<std::string>{devname}));

            }

        }

    }

}



SCENARIO("Telemetry admin commands: logging/tracing endpoints")

{

    int idlver = GENERATE(TangoTest::idlversion(6));



    GIVEN("a device proxy to a simple IDLv" << idlver << " device and a connection to the admin device")

    {

        std::vector<std::string> env{"TANGO_TELEMETRY_ENABLE=off", "TANGO_TELEMETRY_TYPES=tracing,logging"};

        TangoTest::Context ctx{"telemetry", "TelemetryDS", idlver, env};



        auto device = ctx.get_proxy();



        auto admin = std::make_unique<Tango::DeviceProxy>(device->adm_name());

        REQUIRE(admin != nullptr);



        const std::string devname = "TestServer/tests/1";



        Tango::DeviceData ddin, ddout;

        std::vector<std::string> instr, outstr;

        std::vector<Tango::DevLong> inlong, outlong;



        using namespace TangoTest::Matchers;

        using namespace Catch::Matchers;



        std::string telemetry_type = GENERATE("Tracing", "Logging");

        std::string get_command = "GetTelemetry" + telemetry_type + "Endpoints";

        std::string set_command = "SetTelemetry" + telemetry_type + "Endpoints";

        std::string add_command = "AddTelemetry" + telemetry_type + "Endpoints";

        std::string remove_command = "RemoveTelemetry" + telemetry_type + "Endpoints";



        WHEN("we query " + telemetry_type + " endpoints")

        {

            // Check default endpoint set

            instr = {devname};

            ddin << instr;

            ddout = admin->command_inout(get_command, ddin);

            ddout >> outstr;



            THEN("we find the defaults are set")

            {

                CHECK_THAT(outstr, RangeEquals(std::vector<std::string>{devname, "1", "console", "cout"}));

            }

        }



        WHEN("we change the " + telemetry_type + " endpoint to {console, cerr}")

        {

            // Change endpoint to {console, cerr} and check

            instr = {devname, "1", "console", "cerr"};

            ddin << instr;

            ddout = admin->command_inout(set_command, ddin);

            ddout.reset_exceptions(Tango::DeviceData::isempty_flag);

            REQUIRE(ddout.is_empty());



            instr = {devname};

            ddin << instr;

            ddout = admin->command_inout(get_command, ddin);

            ddout >> outstr;



            THEN("we find the endpoint has been updated")

            {

                CHECK_THAT(outstr, RangeEquals(std::vector<std::string>{devname, "1", "console", "cerr"}));

            }

        }



        WHEN("we add the " + telemetry_type + " endpoint {console, cout}")

        {

            // Change endpoint to {console, cerr} and check

            instr = {devname, "1", "console", "cerr"};

            ddin << instr;

            ddout = admin->command_inout(set_command, ddin);

            ddout.reset_exceptions(Tango::DeviceData::isempty_flag);

            REQUIRE(ddout.is_empty());



            // Add endpoint and check for {console, cerr},{console, cout}

            instr = {devname, "console", "cout"};

            ddin << instr;

            ddout = admin->command_inout(add_command, ddin);

            ddout.reset_exceptions(Tango::DeviceData::isempty_flag);

            REQUIRE(ddout.is_empty());



            instr = {devname};

            ddin << instr;

            ddout = admin->command_inout(get_command, ddin);

            ddout >> outstr;



            THEN("we find the endpoint has been added")

            {

                CHECK_THAT(outstr,

                           RangeEquals(std::vector<std::string>{devname, "2", "console", "cerr", "console", "cout"}));

            }

        }



        WHEN("we remove the " + telemetry_type + " endpoint {console, cerr}")

        {

            // Change endpoint to {console, cerr}

            instr = {devname, "1", "console", "cerr"};

            ddin << instr;

            ddout = admin->command_inout(set_command, ddin);

            ddout.reset_exceptions(Tango::DeviceData::isempty_flag);

            REQUIRE(ddout.is_empty());



            // Add endpoint

            instr = {devname, "console", "cout"};

            ddin << instr;

            ddout = admin->command_inout(add_command, ddin);

            ddout.reset_exceptions(Tango::DeviceData::isempty_flag);

            REQUIRE(ddout.is_empty());



            // Remove endpoint and check for {console, cout}

            instr = {devname, "console", "cerr"};

            ddin << instr;

            ddout = admin->command_inout(remove_command, ddin);

            ddout.reset_exceptions(Tango::DeviceData::isempty_flag);

            REQUIRE(ddout.is_empty());



            instr = {devname};

            ddin << instr;

            ddout = admin->command_inout(get_command, ddin);

            ddout >> outstr;



            CHECK_THAT(outstr, RangeEquals(std::vector<std::string>{devname, "1", "console", "cout"}));

        }

    }

}



SCENARIO("Telemetry admin commands: topics")

{

    int idlver = GENERATE(TangoTest::idlversion(6));



    GIVEN("a device proxy to a simple IDLv" << idlver << " device and a connection to the admin device")

    {

        std::vector<std::string> env{"TANGO_TELEMETRY_ENABLE=off", "TANGO_TELEMETRY_TYPES=tracing,logging"};

        TangoTest::Context ctx{"telemetry", "TelemetryDS", idlver, env};



        auto device = ctx.get_proxy();



        auto admin = std::make_unique<Tango::DeviceProxy>(device->adm_name());

        REQUIRE(admin != nullptr);



        const std::string devname = "TestServer/tests/1";



        Tango::DeviceData ddin, ddout;

        std::vector<std::string> instr, outstr;

        std::vector<Tango::DevLong> inlong, outlong;



        using namespace TangoTest::Matchers;

        using namespace Catch::Matchers;



        WHEN("we query topics")

        {

            instr = {devname};

            ddin << instr;

            ddout = admin->command_inout("GetTelemetryTopics", ddin);

            ddout >> outstr;



            REQUIRE(outstr.size() == 3);



            THEN("we get the default")

            {

                CHECK_THAT(outstr, RangeEquals(std::vector<std::string>{devname, "1", "all"}));

            }

        }



        WHEN("we set topics")

        {

            instr = {devname, "2", "polling", "user"};

            ddin << instr;

            ddout = admin->command_inout("SetTelemetryTopics", ddin);

            ddout.reset_exceptions(Tango::DeviceData::isempty_flag);

            REQUIRE(ddout.is_empty());



            instr = {devname};

            ddin << instr;

            ddout = admin->command_inout("GetTelemetryTopics", ddin);

            ddout >> outstr;



            THEN("the topics are updated")

            {

                CHECK_THAT(outstr, UnorderedRangeEquals(std::vector<std::string>{devname, "2", "polling", "user"}));

            }

        }

    }

}



SCENARIO("Telemetry admin commands: exceptions")

{

    int idlver = GENERATE(TangoTest::idlversion(6));



    GIVEN("a device proxy to a simple IDLv" << idlver << " device and a connection to the admin device")

    {

        std::vector<std::string> env{"TANGO_TELEMETRY_ENABLE=off", "TANGO_TELEMETRY_TYPES=tracing,logging"};

        TangoTest::Context ctx{"telemetry", "TelemetryDS", idlver, env};



        auto device = ctx.get_proxy();



        auto admin = std::make_unique<Tango::DeviceProxy>(device->adm_name());

        REQUIRE(admin != nullptr);



        const std::string devname = "TestServer/tests/1";



        Tango::DeviceData ddin, ddout;

        std::vector<std::string> instr, outstr;

        std::vector<Tango::DevLong> inlong, outlong;



        WHEN("invalid devices are queried")

        {

            using namespace Catch::Matchers;

            std::string command = GENERATE("StartTelemetry",

                                           "StopTelemetry",

                                           "IsTelemetryEnabled",

                                           "GetTelemetryTopics",

                                           "GetTelemetryTracing",

                                           "GetTelemetryLogging",

                                           "GetTelemetryTracingEndpoints",

                                           "GetTelemetryLoggingEndpoints");



            instr = {"fakedevicename"};

            ddin << instr;

            CHECK_THROWS_WITH(admin->command_inout(command, ddin), ContainsSubstring("API_DeviceNotFound"));

        }



        WHEN("incorrect input is given")

        {

            std::string telemetry_type = GENERATE("Tracing", "Logging");

            std::string set_command = "SetTelemetry" + telemetry_type + "Endpoints";

            std::string add_command = "AddTelemetry" + telemetry_type + "Endpoints";

            std::string remove_command = "RemoveTelemetry" + telemetry_type + "Endpoints";



            using namespace Catch::Matchers;

            instr = {devname, "one", "console", "cout"};

            ddin << instr;

            CHECK_THROWS_WITH(admin->command_inout(set_command, ddin), ContainsSubstring("API_InvalidArgs"));



            instr = {devname, "1", "console", "cout", "extra"};

            ddin << instr;

            CHECK_THROWS_WITH(admin->command_inout(set_command, ddin), ContainsSubstring("API_InvalidArgs"));



            instr = {devname, "2", "console", "cout"};

            ddin << instr;

            CHECK_THROWS_WITH(admin->command_inout(set_command, ddin), ContainsSubstring("API_InvalidArgs"));



            instr = {devname, "1", "console", "cout"};

            ddin << instr;

            CHECK_THROWS_WITH(admin->command_inout(add_command, ddin), ContainsSubstring("API_InvalidArgs"));



            instr = {devname, "1", "console", "cout"};

            ddin << instr;

            CHECK_THROWS_WITH(admin->command_inout(remove_command, ddin), ContainsSubstring("API_InvalidArgs"));

        }

    }

}



SCENARIO("Telemetry topics are respected")

{

    int idlver = GENERATE(TangoTest::idlversion(6));

    GIVEN("a device proxy to a simple IDLv" << idlver << " device")

    {

        std::vector<std::string> env{"TANGO_TELEMETRY_ENABLE=on", "TANGO_TELEMETRY_TYPES=tracing"};

        TangoTest::Context ctx{"telemetry", "TelemetryDS", idlver, env};



        auto device = ctx.get_proxy();



        WHEN("we read the attribute with default topics")

        {

            std::string att{"attr_dq_db"};



            Tango::DeviceAttribute da;

            REQUIRE_NOTHROW(da = device->read_attribute(att));



            double att_value;

            da >> att_value;

            REQUIRE(att_value == SERVER_VALUE);



            // Force spans to flush

            device->set_telemetry_tracing(false);

            auto contents = load_file(ctx.get_redirect_file());

            REQUIRE(!contents.empty());



            using namespace Catch::Matchers;

            THEN("the telemetry span is exported")

            {

                REQUIRE_THAT(contents, ContainsSubstring("myKey"));

            }

        }



        device->set_telemetry_topics({"user"});

        REQUIRE(device->get_telemetry_topics() == std::vector<std::string>{"user"});



        WHEN("we read the attribute after setting the topic to user")

        {

            std::string att{"attr_dq_db"};



            Tango::DeviceAttribute da;

            REQUIRE_NOTHROW(da = device->read_attribute(att));



            double att_value;

            da >> att_value;

            REQUIRE(att_value == SERVER_VALUE);



            // Force spans to flush

            device->set_telemetry_tracing(false);

            auto contents = load_file(ctx.get_redirect_file());

            REQUIRE(!contents.empty());



            using namespace Catch::Matchers;

            THEN("the telemetry span is exported")

            {

                REQUIRE_THAT(contents, ContainsSubstring("myKey"));

            }

        }



        device->set_telemetry_topics({"polling"});

        REQUIRE(device->get_telemetry_topics() == std::vector<std::string>{"polling"});



        WHEN("we read the attribute after setting not-used topic (misc)")

        {

            std::string att{"attr_dq_db"};



            Tango::DeviceAttribute da;

            REQUIRE_NOTHROW(da = device->read_attribute(att));



            double att_value;

            da >> att_value;

            REQUIRE(att_value == SERVER_VALUE);



            // Force spans to flush

            device->set_telemetry_tracing(false);

            auto contents = load_file(ctx.get_redirect_file());

            REQUIRE(!contents.empty());



            using namespace Catch::Matchers;

            THEN("the telemetry span is not exported")

            {

                REQUIRE_THAT(contents, !ContainsSubstring("myKey"));

            }

        }

    }

}



SCENARIO("Telemetry DeviceProxy commands")

{

    int idlver = GENERATE(TangoTest::idlversion(6));

    GIVEN("a device proxy to a simple IDLv" << idlver << " device")

    {

        std::vector<std::string> env{"TANGO_TELEMETRY_ENABLE=off", "TANGO_TELEMETRY_TYPES=tracing,logging"};

        TangoTest::Context ctx{"telemetry", "TelemetryDS", idlver, env};



        auto device = ctx.get_proxy();

        WHEN("query telemetry status before and after turning telemetry on/off")

        {

            REQUIRE(!device->is_telemetry_enabled());

            device->start_telemetry();

            REQUIRE(device->is_telemetry_enabled());

            device->stop_telemetry();

            REQUIRE(!device->is_telemetry_enabled());



            REQUIRE(device->get_telemetry_tracing());

            device->set_telemetry_tracing(false);

            REQUIRE(!device->get_telemetry_tracing());

            device->set_telemetry_tracing(true);

            REQUIRE(device->get_telemetry_tracing());



            REQUIRE(device->get_telemetry_logging());

            device->set_telemetry_logging(false);

            REQUIRE(!device->get_telemetry_logging());

            device->set_telemetry_logging(true);

            REQUIRE(device->get_telemetry_logging());

        }



        WHEN("update tracing endpoints")

        {

            auto result = device->get_telemetry_tracing_endpoints();

            REQUIRE(result == std::vector<std::string>{"console", "cout"});

            device->set_telemetry_tracing_endpoints({"console", "cerr"});

            result = device->get_telemetry_tracing_endpoints();

            REQUIRE(result == std::vector<std::string>{"console", "cerr"});

            device->add_telemetry_tracing_endpoint({"console", "cout"});

            result = device->get_telemetry_tracing_endpoints();

            REQUIRE(result == std::vector<std::string>{"console", "cerr", "console", "cout"});

            device->remove_telemetry_tracing_endpoint({"console", "cerr"});

            result = device->get_telemetry_tracing_endpoints();

            REQUIRE(result == std::vector<std::string>{"console", "cout"});

        }

        WHEN("update logging endpoints")

        {

            auto result = device->get_telemetry_logging_endpoints();

            REQUIRE(result == std::vector<std::string>{"console", "cout"});

            device->set_telemetry_logging_endpoints({"console", "cerr"});

            result = device->get_telemetry_logging_endpoints();

            REQUIRE(result == std::vector<std::string>{"console", "cerr"});

            device->add_telemetry_logging_endpoint({"console", "cout"});

            result = device->get_telemetry_logging_endpoints();

            REQUIRE(result == std::vector<std::string>{"console", "cerr", "console", "cout"});

            device->remove_telemetry_logging_endpoint({"console", "cerr"});

            result = device->get_telemetry_logging_endpoints();

            REQUIRE(result == std::vector<std::string>{"console", "cout"});

        }

    }

}



SCENARIO("Telemetry properties override environment variables")

{

    int idlver = GENERATE(TangoTest::idlversion(6));

    GIVEN("a device proxy to a simple IDLv" << idlver << " device")

    {

        std::vector<std::string> env{"TANGO_TELEMETRY_ENABLE=off",

                                     "TANGO_TELEMETRY_TYPES=none",

                                     "TANGO_TELEMETRY_TOPICS=user",

                                     "TANGO_TELEMETRY_TRACING_EXPORTERS=console",

                                     "TANGO_TELEMETRY_TRACING_ENDPOINTS=cerr",

                                     "TANGO_TELEMETRY_LOGGING_EXPORTERS=grpc,console",

                                     "TANGO_TELEMETRY_LOGGING_ENDPOINTS=grpc://localhost:554,cout"};

        std::string device_properties = "testserver/tests/1->telemetry_enable: true\n \

                                        testserver/tests/1->telemetry_types: \"logging, tracing\"\n \

                                        testserver/tests/1->telemetry_topics: \"events\"\n \

                                        testserver/tests/1->telemetry_tracing_exporters: \"grpc, console\"\n \

                                        testserver/tests/1->telemetry_tracing_endpoints: \"grpc://localhost:554, cerr\"\n \

                                        testserver/tests/1->telemetry_logging_exporters: \"http\"\n \

                                        testserver/tests/1->telemetry_logging_endpoints: \"http://localhost:3000\"\n";

        TangoTest::Context ctx{"telemetry", "TelemetryDS", idlver, device_properties, env};



        auto device = ctx.get_proxy();



        CHECK(device->is_telemetry_enabled());

        CHECK(device->get_telemetry_tracing());

        CHECK(device->get_telemetry_logging());

        CHECK(device->get_telemetry_topics() == std::vector<std::string>{"events"});

        CHECK(device->get_telemetry_tracing_endpoints() ==

              std::vector<std::string>{"grpc", "grpc://localhost:554", "console", "cerr"});

        CHECK(device->get_telemetry_logging_endpoints() == std::vector<std::string>{"http", "http://localhost:3000"});

    }

}



SCENARIO("Telemetry DS can be restarted without memory leak")

{

    int idlver = GENERATE(TangoTest::idlversion(6));