Commit 898310b0 authored by Jan Oliver Oelerich's avatar Jan Oliver Oelerich

In an attempt to unclutter the code, removed the http reporting feature

parent 8b27365f
......@@ -73,14 +73,6 @@ find_package(NetCDF REQUIRED)
include_directories(${NETCDF_INCLUDE_DIR})
set(LIBS ${LIBS} ${NETCDF_LIBRARIES})
# look for CURL
find_package(CURL)
if(CURL_FOUND)
include_directories(${CURL_INCLUDE_DIRS})
set(LIBS ${LIBS} ${CURL_LIBRARIES})
set(HAVE_CURL 1)
endif(CURL_FOUND)
# MPI
find_package(MPI REQUIRED)
include_directories(${MPI_INCLUDE_PATH})
......
......@@ -42,7 +42,6 @@ set(INTERNAL_FILES
utilities/output.hpp
utilities/algorithms.hpp
utilities/TaskQueue.hpp
utilities/http.hpp
utilities/timings.hpp)
set(EXTERNAL_FILES
......
......@@ -341,13 +341,6 @@ void IO::writeParams(const shared_ptr<GridManager> &gridman) {
g.att("fixed_slicing", p.isFixedSlicing());
}
{
auto g = pg.defineGroup("http_reporting");
g.att("reporting", p.httpReporting());
g.att("url", p.httpUrl());
// the other parameters are not written to output file for security.
}
} catch(NcException &e) {
output::error("%s\n", e.what());
}
......
......@@ -169,8 +169,6 @@ void Params::readParamsFromString(const string &prms) {
output::error("Parse error at %s:%d - %s", pex.getFile(), pex.getLine(), pex.getError());
}
Setting &root = _cfg.getRoot();
//Setting &root = _cfg.getRoot();
_cfg.lookupValue("application.random_seed", _random_seed);
_cfg.lookupValue("application.skip_simulation", _skip_simulation);
......@@ -294,16 +292,6 @@ void Params::readParamsFromString(const string &prms) {
if(_cfg.exists("probe.astigmatism_ca"))
_probe_astigmatism_ca *= 10;
_cfg.lookupValue("http_reporting.reporting", _http_reporting);
_cfg.lookupValue("http_reporting.url", _http_url);
_cfg.lookupValue("http_reporting.auth_user", _http_auth_user);
_cfg.lookupValue("http_reporting.auth_pass", _http_auth_pass);
if(_cfg.exists("http_reporting.parameters")) {
for(auto &prm: root["http_reporting"]["parameters"]) {
_http_parameters.push_back(make_tuple(prm.getName(), (const char *) (prm)));
}
}
if(_tmp_dir.empty()) {
// use the path of the output file directory for tmp
int found = _output_file.find_last_of("/\\");
......@@ -398,11 +386,6 @@ void Params::broadcast(int rank) {
mpi_env.broadcast(_adf_average_defoci, rank);
mpi_env.broadcast(_adf_save_every_n_slices, rank);
mpi_env.broadcast(_http_reporting, rank);
mpi_env.broadcast(_http_url, rank);
mpi_env.broadcast(_http_auth_user, rank);
mpi_env.broadcast(_http_auth_pass, rank);
}
void Params::readParamsFromNCFile() {
......
......@@ -340,46 +340,6 @@ namespace stemsalabim {
return _diagnostics_dir;
}
/*!
* Return whether HTTP reporting is enabled.
* @return true if HTTP reporting is ON.
*/
bool httpReporting() const {
return _http_reporting;
}
/*!
* Vector of key/value pairs that are sent as additional payload in HTTP reporting requests.
* @return vector of additional key/value tuples in HTTP reporting requests.
*/
const std::vector<std::tuple<std::string, std::string>> &httpParameters() const {
return _http_parameters;
}
/*!
* Return the HTTP reporting endpoint (URL).
* @return HTTP reporting URL
*/
const std::string &httpUrl() const {
return _http_url;
}
/*!
* Return the username for HTTP reporting basic auth.
* @return username of HTTP basic auth
*/
const std::string &httpAuthUser() const {
return _http_auth_user;
}
/*!
* Return the password for HTTP reporting basic auth.
* @return password of HTTP basic auth
*/
const std::string &httpAuthPass() const {
return _http_auth_pass;
}
/*!
* Return whether the multi-slice simulation is skipped, i.e., a dry-run is performed
* writing only crystal coordinates to the output.
......@@ -673,12 +633,6 @@ namespace stemsalabim {
bool _cbed_average_configurations{true};
bool _cbed_average_defoci{true};
bool _http_reporting{false};
std::string _http_url{""};
std::string _http_auth_user{""};
std::string _http_auth_pass{""};
std::vector<std::tuple<std::string, std::string>> _http_parameters;
libconfig::Config _cfg;
std::vector<std::string> _cli_options;
......
......@@ -23,7 +23,6 @@
#include <memory>
#include <thread>
#include "../libatomic/Scattering.hpp"
#include "../utilities/http.hpp"
#include "../utilities/timings.hpp"
using namespace std;
......@@ -52,18 +51,6 @@ void Simulation::init() {
p.numberOfThreads());
{
ifstream t("/proc/" + to_string(getpid()) + "/status");
string procstatus((istreambuf_iterator<char>(t)), istreambuf_iterator<char>());
istringstream f(procstatus);
string line;
while(getline(f, line)) {
if(line.find("Cpus_allowed_list") != string::npos) {
PRINT_DIAGNOSTICS(output::fmt("[%s, %d] %s.\n", mpi_env.name(), mpi_env.rank(), line));
}
}
}
// read and communicate the content of the crystal file and initialize crystal
// object.
{
......@@ -128,9 +115,6 @@ void Simulation::init() {
_gridman->generateGrids();
// post start signal after generation of the grids.
postStatus(STATUS_SIGNAL::START_SIMULATION, SimulationState() /* actually not required for this signal */);
// generateSlices must be done before writing anything to the output file, i.e.,
// before calling io->init()
generatePropagator();
......@@ -211,11 +195,9 @@ void Simulation::run() {
// if a new defocus was started, do some output stuff.
if(!st.iconf()) {
postStatus(STATUS_SIGNAL::START_DEFOCUS, st);
printMaster(st, output::fmt("Starting Defocus %.2fnm.\n", st.defocus() / 10));
}
postStatus(STATUS_SIGNAL::START_CONFIGURATION, st);
printMaster(st, "Starting configuration. \n");
t1 = high_resolution_clock::now();
......@@ -317,8 +299,6 @@ void Simulation::run() {
};
postStatus(STATUS_SIGNAL::FINISH_SIMULATION, SimulationState());
printMaster(output::fmt("Finished computation in %s. Find the results in %s.\n",
output::humantime(algorithms::getTimeSince(start_sim)),
prms.outputFilename()));
......@@ -569,8 +549,6 @@ void Simulation::multisliceMaster(const SimulationState &st) {
if(p * 100 > last_prcnt) {
last_prcnt = (unsigned int) ceil(p * 100);
postStatus(STATUS_SIGNAL::PROGRESS, st, p);
printLine(st,
output::fmt("progr: %d%%, eta/conf: %s, eta/defocus: %s, eta/sim: %s\n",
ceil(p * 100),
......@@ -611,8 +589,6 @@ void Simulation::multisliceMaster(const SimulationState &st) {
if(p * 100 > last_prcnt) {
last_prcnt = (unsigned int) ceil(p * 100);
postStatus(STATUS_SIGNAL::PROGRESS, st, p);
printLine(st,
output::fmt("progr: %d%%, eta/conf: %s, eta/defocus: %s, eta/sim: %s\n",
ceil(p * 100),
......@@ -846,65 +822,3 @@ void Simulation::makeProbe(Wave &wave, const double defocus, double px, double p
wave.normalize();
wave.backwardFFT();
}
void Simulation::postStatus(STATUS_SIGNAL signal, const SimulationState &st, float progress) {
#if(HAVE_CURL)
Params &p = Params::getInstance();
if(!p.httpReporting())
return;
mpi::Environment &mpi_env = mpi::Environment::getInstance();
if(mpi_env.isSlave())
return;
// cancel request when the last request was less than 20 seconds ago
// TODO: Make this configurable
if(signal == STATUS_SIGNAL::PROGRESS &&
algorithms::getTimeSince(_http_request_timer) < duration_cast<microseconds>(seconds(20)))
return;
http::Environment &e = http::Environment::getInstance();
Json::Value c;
c["time"] = output::currentTimeString();
c["id"] = p.uuid();
c["num_threads"] = p.numberOfThreads();
c["num_processors"] = mpi_env.size();
c["num_defoci"] = _gridman->defoci().size();
c["num_configurations"] = p.numberOfConfigurations();
if(signal == STATUS_SIGNAL::START_SIMULATION) {
c["event"] = "START_SIMULATION";
c["version"] = output::fmt("%s.%s.%s", PKG_VERSION_MAJOR, PKG_VERSION_MINOR, PKG_VERSION_PATCH);
c["git_commit"] = GIT_COMMIT_HASH;
c["title"] = p.title();
} else if(signal == STATUS_SIGNAL::START_DEFOCUS) {
c["event"] = "START_DEFOCUS";
c["defocus"] = st.defocus() / 10.;
c["defocus_index"] = st.idefocus();
} else if(signal == STATUS_SIGNAL::START_CONFIGURATION) {
c["event"] = "START_FP_CONFIGURATION";
c["defocus"] = st.defocus() / 10.;
c["defocus_index"] = st.idefocus();
c["configuration_index"] = st.iconf();
} else if(signal == STATUS_SIGNAL::PROGRESS) {
c["event"] = "PROGRESS";
c["defocus"] = st.defocus() / 10.;
c["defocus_index"] = st.idefocus();
c["configuration_index"] = st.iconf();
c["progress"] = progress;
} else if(signal == STATUS_SIGNAL::FINISH_SIMULATION) {
c["event"] = "FINISH";
}
_http_request_timer = high_resolution_clock::now();
e.post(c);
#endif
}
/*
* STEMsalabim: Magical STEM image simulations
*
* Copyright (c) 2016-2018 Jan Oliver Oelerich <[email protected]>
* Copyright (c) 2016-2018 Structure and Technology Research Laboratory, Philipps-Universität Marburg, Germany
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#ifndef STEMSALABIM_HTTP_HPP
#define STEMSALABIM_HTTP_HPP
/** @file */
#include "config.h"
// This file is only required when libCURL is found.
#if(HAVE_CURL)
#include <tuple>
#include <memory>
#include <string>
#include <curl/curl.h>
#include <vector>
#include "../classes/Params.hpp"
#include "algorithms.hpp"
#include "output.hpp"
namespace stemsalabim {
/*!
* Contains classes and functions related to HTTP requests.
*/
namespace http {
size_t my_dummy_write(char *ptr, size_t size, size_t nmemb, void *userdata) {
(void) (userdata);
(void) (ptr);
return size * nmemb;
}
/*!
* Singleton class to manage the global CURL initialization state.
* An instance needs to be fetched using getInstance.
*/
class Environment {
public:
/*!
* Return the instance of the class, which is created if called for the first time.
* @return http::Environment instance.
*/
static Environment &getInstance() {
static Environment instance;
return instance;
}
/*!
* Submit a JSON POST request with some custom content. The URL, Auth etc. is
* read from the Params object.
* @param content Json node with additional payload to append to the request.
*/
void post(const Json::Value &content) {
Params &prms = Params::getInstance();
CURL *curl;
curl = curl_easy_init();
if(curl) {
Json::Value root;
// Add parameters specified in the configuration file
std::vector<std::string> query_parts;
for(auto &p : prms.httpParameters()) {
root[std::get<0>(p)] = std::get<1>(p);
}
// the custom content will be in the 'content' = {...} node.
root["content"] = content;
// to string
Json::FastWriter fastWriter;
std::string payload = fastWriter.write(root);
// initialize curl request
struct curl_slist *headers = NULL;
headers = curl_slist_append(headers, "Accept: application/json");
headers = curl_slist_append(headers, "Content-Type: application/json");
headers = curl_slist_append(headers, "charsets: utf-8");
curl_easy_setopt(curl, CURLOPT_URL, prms.httpUrl().c_str());
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload.c_str());
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &my_dummy_write);
// TODO: disable SSL verification. This is unsecure, I know, and should be
// moved to an option in the parameter file.
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
if(!prms.httpAuthUser().empty()) {
std::string login = prms.httpAuthUser() + ":" + prms.httpAuthPass();
curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
curl_easy_setopt(curl, CURLOPT_USERPWD, login.c_str());
}
CURLcode res = curl_easy_perform(curl);
// for http basic auth, the first request is for auth and the second request
// actually sends the stuff.
if(!prms.httpAuthUser().empty()) {
res = curl_easy_perform(curl);
}
if(res != CURLE_OK)
output::stderr("HTTP status request failed: %s\n", curl_easy_strerror(res));
curl_easy_cleanup(curl);
}
}
private:
/*!
* Constructor is private to avoid multiple construction
*/
Environment()
: _initialized(false) {
if(!_initialized) {
curl_global_init(CURL_GLOBAL_ALL);
_initialized = true;
}
}
/*!
* Clean up curl.
*/
~Environment() {
if(!_initialized)
return;
curl_global_cleanup();
}
bool _initialized;
public:
Environment(Environment const &) = delete;
void operator=(Environment const &) = delete;
};
}
}
#endif // HAVE_CURL
#endif //STEMSALABIM_HTTP_HPP
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment