diff --git a/Makefile.am b/Makefile.am index 41ca5cf..2144f19 100644 --- a/Makefile.am +++ b/Makefile.am @@ -70,3 +70,7 @@ distsign: distcheck > $(top_srcdir)/$(distdir).tar.gz.md5 sha1sum $(top_srcdir)/$(distdir).tar.gz \ > $(top_srcdir)/$(distdir).tar.gz.sha + +.PHONY: t +t: all + @erl -pa src/couchdb/ -eval "couch_stats_aggregator:test()" -eval "couch_stats_collector:test()" \ No newline at end of file diff --git a/etc/couchdb/default.ini.tpl.in b/etc/couchdb/default.ini.tpl.in index 28ab36f..06e8b36 100644 --- a/etc/couchdb/default.ini.tpl.in +++ b/etc/couchdb/default.ini.tpl.in @@ -33,6 +33,8 @@ external_manager={couch_external_manager, start_link, []} db_update_notifier={couch_db_update_notifier_sup, start_link, []} query_servers={couch_query_servers, start_link, []} httpd={couch_httpd, start_link, []} +stats_aggregator={couch_stats_aggregator, start, []} +stats_collector={couch_stats_collector, start, []} [httpd_global_handlers] / = {couch_httpd_misc_handlers, handle_welcome_req, <<"Welcome">>} @@ -40,12 +42,14 @@ favicon.ico = {couch_httpd_misc_handlers, handle_favicon_req, "%localdatadir%/ww _utils = {couch_httpd_misc_handlers, handle_utils_dir_req, "%localdatadir%/www"} _all_dbs = {couch_httpd_misc_handlers, handle_all_dbs_req} -_stats = {couch_httpd_misc_handlers, handle_stats_req} +;_stats = {couch_httpd_misc_handlers, handle_stats_req} _active_tasks = {couch_httpd_misc_handlers, handle_task_status_req} _config = {couch_httpd_misc_handlers, handle_config_req} _replicate = {couch_httpd_misc_handlers, handle_replicate_req} _uuids = {couch_httpd_misc_handlers, handle_uuids_req} _restart = {couch_httpd_misc_handlers, handle_restart_req} +_stats = {couch_httpd_stats_handlers, handle_stats_req} + [httpd_db_handlers] _view = {couch_httpd_view, handle_view_req} diff --git a/etc/couchdb/local_dev.ini b/etc/couchdb/local_dev.ini index 09123cf..510f5c6 100644 --- a/etc/couchdb/local_dev.ini +++ b/etc/couchdb/local_dev.ini @@ -12,7 +12,10 @@ ;bind_address = 127.0.0.1 [log] -level = error +level = debug [update_notification] ;unique notifier name=/full/path/to/exe -with "cmd line arg" + +[test] +foo = bar diff --git a/share/www/script/couch.js b/share/www/script/couch.js index 0a2698a..7926645 100644 --- a/share/www/script/couch.js +++ b/share/www/script/couch.js @@ -295,6 +295,13 @@ CouchDB.request = function(method, uri, options) { return req; } +CouchDB.requestStats = function(module, key, options) { + options = options || {}; + var stat = CouchDB.request("GET", "/_stats/" + module + "/" + key + + "?" + CouchDB.params(options)).responseText; + return JSON.parse(stat)[module][key]; +} + CouchDB.uuids_cache = []; CouchDB.newUuids = function(n) { @@ -327,3 +334,13 @@ CouchDB.maybeThrowError = function(req) { throw result; } } + +CouchDB.params = function(options) { + options = options || {}; + var returnArray = []; + for(var key in options) { + var value = options[key]; + returnArray.push(key + "=" + value); + } + return returnArray.join("&"); +} \ No newline at end of file diff --git a/share/www/script/couch_test_runner.js b/share/www/script/couch_test_runner.js index 31bd4a4..beede46 100644 --- a/share/www/script/couch_test_runner.js +++ b/share/www/script/couch_test_runner.js @@ -152,13 +152,13 @@ function updateTestsFooter() { // display the line that failed. // Example: // T(MyValue==1); -function T(arg1, arg2) { +function T(arg1, arg2, testName) { if (!arg1) { if (currentRow) { if ($("td.details ol", currentRow).length == 0) { $("
    ").appendTo($("td.details", currentRow)); } - $("
  1. Assertion failed:
    2. ") + $("
  2. Assertion " + (testName ? "'" + testName + "'" : "") + " failed:
    3. ") .find("code").text((arg2 != null ? arg2 : arg1).toString()).end() .appendTo($("td.details ol", currentRow)); } @@ -166,6 +166,10 @@ function T(arg1, arg2) { } } +function TEquals(expected, actual, testName) { + T(equals(expected, actual), "expected '" + expected + "', got '" + actual + "'", testName); +} + function equals(a,b) { if (a === b) return true; try { diff --git a/share/www/script/couch_tests.js b/share/www/script/couch_tests.js index a6c47a7..e1eefd9 100644 --- a/share/www/script/couch_tests.js +++ b/share/www/script/couch_tests.js @@ -21,6 +21,425 @@ if (typeof window == 'undefined' || !window) { var tests = { + stats: function(debug) { + if (debug) debugger; + + var open_databases_tests = { + 'should increment the number of open databases when creating a db': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + var open_databases = parseInt(CouchDB.requestStats("couch_db", "open_databases")); + db.createDb(); + + var new_open_databases = parseInt(CouchDB.requestStats("couch_db", "open_databases")); + TEquals(parseInt(open_databases) + 1, parseInt(new_open_databases), name); + }, + 'should increment the number of open databases when opening a db': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + + restartServer(); + + var open_databases = parseInt(CouchDB.requestStats("couch_db", "open_databases")); + + db.open("123"); + + var new_open_databases = parseInt(CouchDB.requestStats("couch_db", "open_databases")); + TEquals(parseInt(open_databases) + 1, parseInt(new_open_databases), name); + }, + 'should decrement the number of open databases when deleting': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + var open_databases = parseInt(CouchDB.requestStats("couch_db", "open_databases")); + + db.deleteDb(); + var new_open_databases = parseInt(CouchDB.requestStats("couch_db", "open_databases")); + TEquals(parseInt(open_databases) - 1, parseInt(new_open_databases), name); + }, + 'should keep the same number of open databases when reaching the max_dbs_open limit': function(name) { + restartServer(); + var max = 5; + run_on_modified_server( + [{section: "couchdb", + key: "max_dbs_open", + value: max.toString()}], + + function () { + for(var i=0; i= 0, "requests >= 0", name); + TEquals(requests + 1, new_requests, name); + }, + 'should return the average requests/s for the last minute': function(name) { + restartServer(); + var requests = parseFloat(CouchDB.requestStats("httpd", "average_requests")); + TEquals(requests, 0.0, name); + }, + 'should return the average request/s for the last 5 and 15 minutes': function(name) { + restartServer(); + var requests = parseInt(CouchDB.requestStats("httpd", "average_requests"), {"timeframe":5}); + TEquals(requests, 0, name); + + var requests = parseInt(CouchDB.requestStats("httpd", "average_requests"), {"timeframe":15}); + TEquals(requests, 0, name); + } + }; + + var document_read_count_tests = { + 'should increase read document counter when a document is read': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + db.save({"_id":"test"}); + + var reads = parseInt(CouchDB.requestStats("httpd", "document_reads")); + db.open("test"); + var new_reads = parseInt(CouchDB.requestStats("httpd", "document_reads")); + + TEquals(reads + 1 , new_reads, name); + }, + 'should not increase read document counter when a non-document is read': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + db.save({"_id":"test"}); + + var reads = parseInt(CouchDB.requestStats("httpd", "document_reads")); + CouchDB.request("GET", "/"); + var new_reads = parseInt(CouchDB.requestStats("httpd", "document_reads")); + + TEquals(reads, new_reads, name); + }, + 'should increase read document counter when a document\'s revisions are read': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + db.save({"_id":"test"}); + + var reads = parseInt(CouchDB.requestStats("httpd", "document_reads")); + db.open("test", {"open_revs":"all"}); + var new_reads = parseInt(CouchDB.requestStats("httpd", "document_reads")); + + TEquals(reads + 1 , new_reads, name); + } + }; + + var view_read_count_tests = { + 'should increase the permanent view read counter': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + + var reads = parseInt(CouchDB.requestStats("httpd", "view_reads")); + createAndRequestView(db); + var new_reads = parseInt(CouchDB.requestStats("httpd", "view_reads")); + + TEquals(reads + 1 , new_reads, name); + }, + 'should not increase the permanent view read counter when a document is read': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + db.save({"_id":"test"}); + + var reads = parseInt(CouchDB.requestStats("httpd", "view_reads")); + db.open("test"); + var new_reads = parseInt(CouchDB.requestStats("httpd", "view_reads")); + + TEquals(reads, new_reads, name); + }, + 'should not increase the permanent view read counter when a temporary view is read': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + + var reads = parseInt(CouchDB.requestStats("httpd", "view_reads")); + db.query(function(doc) { emit(doc._id)}); + var new_reads = parseInt(CouchDB.requestStats("httpd", "view_reads")); + + TEquals(reads, new_reads, name); + }, + 'should increase the temporary view read counter': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + + var reads = parseInt(CouchDB.requestStats("httpd", "temporary_view_reads")); + db.query(function(doc) { emit(doc._id)}); + var new_reads = parseInt(CouchDB.requestStats("httpd", "temporary_view_reads")); + + TEquals(reads + 1, new_reads, name); + }, + 'should increase the temporary view read counter when querying a permanent view': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + + var reads = parseInt(CouchDB.requestStats("httpd", "view_reads")); + createAndRequestView(db); + var new_reads = parseInt(CouchDB.requestStats("httpd", "view_reads")); + + TEquals(reads + 1 , new_reads, name); + } + }; + + var http_requests_by_method_tests = { + 'should count GET requests': function(name) { + var requests = parseInt(CouchDB.requestStats("httpd", "get_requests")); + var new_requests = parseInt(CouchDB.requestStats("httpd", "get_requests")); + + TEquals(requests + 1, new_requests, name); + }, + 'should not count GET requests for POST request': function(name) { + var requests = parseInt(CouchDB.requestStats("httpd", "get_requests")); + CouchDB.request("POST", "/"); + var new_requests = parseInt(CouchDB.requestStats("httpd", "get_requests")); + + TEquals(requests + 1, new_requests, name); + }, + 'should count POST requests': function(name) { + var requests = parseInt(CouchDB.requestStats("httpd", "post_requests")); + CouchDB.request("POST", "/"); + var new_requests = parseInt(CouchDB.requestStats("httpd", "post_requests")); + + TEquals(requests + 1, new_requests, name); + } + }; + + var document_write_count_tests = { + 'should increment counter for document creates': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + + var creates = parseInt(CouchDB.requestStats("httpd", "document_creates")); + db.save({"a":"1"}); + var new_creates = parseInt(CouchDB.requestStats("httpd", "document_creates")); + + TEquals(creates + 1, new_creates, name); + }, + 'should not increment counter for document creates when updating a doc': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + + var doc = {"_id":"test"}; + db.save(doc); + + var creates = parseInt(CouchDB.requestStats("httpd", "document_creates")); + db.save(doc); + var new_creates = parseInt(CouchDB.requestStats("httpd", "document_creates")); + + TEquals(creates, new_creates, name); + }, + 'should increment counter for document updates': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + + var doc = {"_id":"test"}; + db.save(doc); + + var updates = parseInt(CouchDB.requestStats("httpd", "document_updates")); + db.save(doc); + var new_updates = parseInt(CouchDB.requestStats("httpd", "document_updates")); + + TEquals(updates + 1, new_updates, name); + }, + 'should not increment counter for document updates when creating a document': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + + var updates = parseInt(CouchDB.requestStats("httpd", "document_updates")); + db.save({"a":"1"}); + var new_updates = parseInt(CouchDB.requestStats("httpd", "document_updates")); + + TEquals(updates, new_updates, name); + }, + 'should increment counter for document deletes': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + + var doc = {"_id":"test"}; + db.save(doc); + + var deletes = parseInt(CouchDB.requestStats("httpd", "document_deletes")); + db.deleteDoc(doc); + var new_deletes = parseInt(CouchDB.requestStats("httpd", "document_deletes")); + + TEquals(deletes + 1, new_deletes, name); + }, + 'should increment the copy counter': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + + var doc = {"_id":"test"}; + db.save(doc); + + var copies = parseInt(CouchDB.requestStats("httpd", "document_copies")); + CouchDB.request("COPY", "/test_suite_db/test", { + headers: {"Destination":"copy_of_test"} + }); + var new_copies = parseInt(CouchDB.requestStats("httpd", "document_copies")); + + TEquals(copies + 1, new_copies, name); + }, + 'should increment the move counter': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + + var doc = {"_id":"test"}; + db.save(doc); + + var moves = parseInt(CouchDB.requestStats("httpd", "document_moves")); + CouchDB.request("MOVE", "/test_suite_db/test?rev=" + doc._rev, { + headers: {"Destination":"move_of_test"} + }); + var new_moves = parseInt(CouchDB.requestStats("httpd", "document_moves")); + + TEquals(moves + 1, new_moves, name); + }, + 'should increase the bulk doc counter': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + + var bulks = parseInt(CouchDB.requestStats("httpd", "bulk_requests")); + + var docs = makeDocs(5); + db.bulkSave(docs); + + var new_bulks = parseInt(CouchDB.requestStats("httpd", "bulk_requests")); + + TEquals(bulks + 1, new_bulks, name); + }, + 'should increment counter for document creates using POST': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + + var creates = parseInt(CouchDB.requestStats("httpd", "document_creates")); + CouchDB.request("POST", "/test_suite_db", {body:'{"a":"1"}'}); + var new_creates = parseInt(CouchDB.requestStats("httpd", "document_creates")); + + TEquals(creates + 1, new_creates, name); + }, + 'should increment document create counter when adding attachment': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + + var creates = parseInt(CouchDB.requestStats("httpd", "document_creates")); + CouchDB.request("PUT", "/test_suite_db/bin_doc2/foo2.txt", { + body:"This is no base64 encoded text", + headers:{"Content-Type": "text/plain;charset=utf-8"} + }); + var new_creates = parseInt(CouchDB.requestStats("httpd", "document_creates")); + TEquals(creates + 1, new_creates, name); + }, + 'should increment document update counter when adding attachment to existing doc': function(name) { + var db = new CouchDB("test_suite_db"); + db.deleteDb(); + db.createDb(); + + var doc = {_id:"test"}; + db.save(doc); + + var updates = parseInt(CouchDB.requestStats("httpd", "document_updates")); + CouchDB.request("PUT", "/test_suite_db/test/foo2.txt?rev=" + doc._rev, { + body:"This is no base64 encoded text", + headers:{"Content-Type": "text/plain;charset=utf-8"} + }); + var new_updates = parseInt(CouchDB.requestStats("httpd", "document_updates")); + TEquals(updates + 1, new_updates, name); + } + + }; + var response_codes_tests = { + 'should increment the response code counter': function(name) { + var db = new CouchDB("nonexistant_db"); + db.deleteDb(); + + var not_founds = parseInt(CouchDB.requestStats("http_status_codes", "404")); + CouchDB.request("GET", "/nonexistant_db"); + var new_not_founds = parseInt(CouchDB.requestStats("http_status_codes", "404")); + + TEquals(not_founds + 1, new_not_founds, name); + }, + 'should not increment respinse code counter for other response code': function(name) { + var not_founds = parseInt(CouchDB.requestStats("http_status_codes", "404")); + CouchDB.request("GET", "/"); + var new_not_founds = parseInt(CouchDB.requestStats("http_status_codes", "404")); + + TEquals(not_founds, new_not_founds, name); + } + + }; + + var tests = [ + open_databases_tests, + request_count_tests, + document_read_count_tests, + view_read_count_tests, + http_requests_by_method_tests, + document_write_count_tests, + response_codes_tests + ]; + + for(var testGroup in tests) { + for(var test in tests[testGroup]) { + tests[testGroup][test](test); + } + }; + + function createAndRequestView(db) { + var designDoc = { + _id:"_design/test", // turn off couch.js id escaping? + language: "javascript", + views: { + all_docs_twice: {map: "function(doc) { emit(doc.integer, null); emit(doc.integer, null) }"}, + } + }; + db.save(designDoc); + + db.view("test/all_docs_twice"); + } + + }, + // Do some basic tests. basics: function(debug) { var result = JSON.parse(CouchDB.request("GET", "/").responseText); @@ -2667,7 +3086,7 @@ var tests = { } }) }) - } + }, }; T(db.save(designDoc).ok); diff --git a/src/couchdb/Makefile.am b/src/couchdb/Makefile.am index 1ad5d14..769cd1c 100644 --- a/src/couchdb/Makefile.am +++ b/src/couchdb/Makefile.am @@ -58,6 +58,7 @@ source_files = \ couch_httpd_show.erl \ couch_httpd_view.erl \ couch_httpd_misc_handlers.erl \ + couch_httpd_stats_handlers.erl \ couch_key_tree.erl \ couch_log.erl \ couch_os_process.erl \ @@ -65,6 +66,8 @@ source_files = \ couch_rep.erl \ couch_server.erl \ couch_server_sup.erl \ + couch_stats_aggregator.erl \ + couch_stats_collector.erl \ couch_stream.erl \ couch_task_status.erl \ couch_util.erl \ @@ -95,6 +98,7 @@ compiled_files = \ couch_httpd_show.beam \ couch_httpd_view.beam \ couch_httpd_misc_handlers.beam \ + couch_httpd_stats_handlers.beam \ couch_key_tree.beam \ couch_log.beam \ couch_os_process.beam \ @@ -102,6 +106,8 @@ compiled_files = \ couch_rep.beam \ couch_server.beam \ couch_server_sup.beam \ + couch_stats_aggregator.beam \ + couch_stats_collector.beam \ couch_stream.beam \ couch_task_status.beam \ couch_util.beam \ @@ -150,7 +156,7 @@ couch.app: couch.app.tpl # $(ERL) -noshell -run edoc_run files [\"$<\"] %.beam: %.erl couch_db.hrl - $(ERLC) $< + $(ERLC) -D$TEST $< install-data-hook: if test -f "$(DESTDIR)/$(couchprivlibdir)/couch_erl_driver"; then \ diff --git a/src/couchdb/couch_httpd.erl b/src/couchdb/couch_httpd.erl index 7c37132..a9a130b 100644 --- a/src/couchdb/couch_httpd.erl +++ b/src/couchdb/couch_httpd.erl @@ -126,11 +126,8 @@ handle_request(MochiReq, UrlHandlers, DbUrlHandlers) -> mochiweb_headers:to_list(MochiReq:get(headers)) ]), - Method = + Method1 = case MochiReq:get(method) of - % alias HEAD to GET as mochiweb takes care of stripping the body - 'HEAD' -> 'GET'; - % already an atom Meth when is_atom(Meth) -> Meth; @@ -138,6 +135,15 @@ handle_request(MochiReq, UrlHandlers, DbUrlHandlers) -> % possible (if any module references the atom, then it's existing). Meth -> couch_util:to_existing_atom(Meth) end, + + increment_method_stats(Method1), + + % alias HEAD to GET as mochiweb takes care of stripping the body + Method = case Method1 of + 'HEAD' -> 'GET'; + Other -> Other + end, + HttpReq = #httpd{ mochi_req = MochiReq, method = Method, @@ -166,8 +172,12 @@ handle_request(MochiReq, UrlHandlers, DbUrlHandlers) -> RawUri, Resp:get(code) ]), + couch_stats_collector:increment({httpd, request_count}), {ok, Resp}. +increment_method_stats(Method) -> + CounterName = list_to_atom(string:to_lower(atom_to_list(Method)) ++ "_requests"), + couch_stats_collector:increment({httpd, CounterName}). special_test_authentication_handler(Req) -> case header_value(Req, "WWW-Authenticate") of @@ -319,6 +329,7 @@ basic_username_pw(Req) -> start_chunked_response(#httpd{mochi_req=MochiReq}, Code, Headers) -> + couch_stats_collector:increment({http_status_codes, Code}), {ok, MochiReq:respond({Code, Headers ++ server_header(), chunked})}. send_chunk(Resp, Data) -> @@ -326,6 +337,7 @@ send_chunk(Resp, Data) -> {ok, Resp}. send_response(#httpd{mochi_req=MochiReq}, Code, Headers, Body) -> + couch_stats_collector:increment({http_status_codes, Code}), if Code >= 400 -> ?LOG_DEBUG("HTTPd ~p error response:~n ~s", [Code, Body]); true -> ok diff --git a/src/couchdb/couch_httpd_db.erl b/src/couchdb/couch_httpd_db.erl index 2cb4c40..c66c903 100644 --- a/src/couchdb/couch_httpd_db.erl +++ b/src/couchdb/couch_httpd_db.erl @@ -80,6 +80,7 @@ db_req(#httpd{method='POST',path_parts=[_DbName]}=Req, Db) -> Doc = couch_doc:from_json_obj(couch_httpd:json_body(Req)), DocId = couch_util:new_uuid(), {ok, NewRev} = couch_db:update_doc(Db, Doc#doc{id=DocId, revs=[]}, []), + couch_stats_collector:increment({httpd, document_creates}), send_json(Req, 201, {[ {ok, true}, {id, DocId}, @@ -100,6 +101,7 @@ db_req(#httpd{path_parts=[_,<<"_ensure_full_commit">>]}=Req, _Db) -> send_method_not_allowed(Req, "POST"); db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>]}=Req, Db) -> + couch_stats_collector:increment({httpd, bulk_requests}), {JsonProps} = couch_httpd:json_body(Req), DocsArray = proplists:get_value(<<"docs">>, JsonProps), case couch_httpd:header_value(Req, "X-Couch-Full-Commit", "false") of @@ -370,6 +372,7 @@ db_doc_req(#httpd{method='DELETE'}=Req, Db, DocId) -> couch_httpd:send_error(Req, 412, <<"missing_rev">>, <<"Document rev/etag must be specified to delete">>); RevToDelete -> + couch_stats_collector:increment({httpd, document_deletes}), {ok, NewRev} = couch_db:delete_doc(Db, DocId, [RevToDelete]), send_json(Req, 200, {[ {ok, true}, @@ -393,7 +396,8 @@ db_doc_req(#httpd{method='GET'}=Req, Db, DocId) -> [] -> [{"Etag", DiskEtag}]; % output etag only when we have no meta _ -> [] end, - send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)) + couch_stats_collector:increment({httpd, document_reads}), + send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)) end); _ -> {ok, Results} = couch_db:open_doc_revs(Db, DocId, Revs, Options), @@ -416,6 +420,7 @@ db_doc_req(#httpd{method='GET'}=Req, Db, DocId) -> end, "", Results), send_chunk(Resp, "]"), + couch_stats_collector:increment({httpd, document_reads}), end_json_response(Resp) end; @@ -460,8 +465,10 @@ db_doc_req(#httpd{method='PUT'}=Req, Db, DocId) -> end, case extract_header_rev(Req, ExplicitRev) of missing_rev -> + couch_stats_collector:increment({httpd, document_creates}), Revs = []; Rev -> + couch_stats_collector:increment({httpd, document_updates}), Revs = [Rev] end, {ok, NewRev} = couch_db:update_doc(Db, Doc#doc{id=DocId, revs=Revs}, Options), @@ -485,6 +492,7 @@ db_doc_req(#httpd{method='COPY'}=Req, Db, SourceDocId) -> % save new doc {ok, NewTargetRev} = couch_db:update_doc(Db, Doc#doc{id=TargetDocId, revs=TargetRev}, []), + couch_stats_collector:increment({httpd, document_copies}), send_json(Req, 201, [{"Etag", "\"" ++ binary_to_list(NewTargetRev) ++ "\""}], {[ {ok, true}, @@ -510,9 +518,9 @@ db_doc_req(#httpd{method='MOVE'}=Req, Db, SourceDocId) -> Doc#doc{id=TargetDocId, revs=TargetRev}, #doc{id=SourceDocId, revs=[SourceRev], deleted=true} ], - {ok, ResultRevs} = couch_db:update_docs(Db, Docs, []), - + couch_stats_collector:increment({httpd, document_moves}), + DocResults = lists:zipwith( fun(FDoc, NewRev) -> {[{id, FDoc#doc.id}, {rev, NewRev}]} @@ -602,8 +610,10 @@ db_attachment_req(#httpd{method=Method}=Req, Db, DocId, FileNameParts) Doc = case extract_header_rev(Req, couch_httpd:qs_value(Req, "rev")) of missing_rev -> % make the new doc + couch_stats_collector:increment({httpd, document_creates}), #doc{id=DocId}; Rev -> + couch_stats_collector:increment({httpd, document_updates}), case couch_db:open_doc_revs(Db, DocId, [Rev], []) of {ok, [{ok, Doc0}]} -> Doc0#doc{revs=[Rev]}; {ok, [Error]} -> throw(Error) diff --git a/src/couchdb/couch_httpd_stats_handlers.erl b/src/couchdb/couch_httpd_stats_handlers.erl new file mode 100644 index 0000000..f307582 --- /dev/null +++ b/src/couchdb/couch_httpd_stats_handlers.erl @@ -0,0 +1,30 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_httpd_stats_handlers). +-include("couch_db.hrl"). + +-export([handle_stats_req/1]). +-import(couch_httpd, + [send_json/2,send_json/3,send_json/4,send_method_not_allowed/2, + start_json_response/2,send_chunk/2,end_json_response/1, + start_chunked_response/3, send_error/4]). + +handle_stats_req(#httpd{method='GET', path_parts=PathParts}=Req) -> + [_Db, Module, Key] = PathParts, + Options = couch_httpd:qs(Req), + + Count = couch_stats_aggregator:get({Module, Key}, Options), + Response = {[{Module, {[{Key, Count}]}}]}, + send_json(Req, Response); +handle_stats_req(Req) -> + send_method_not_allowed(Req, "GET"). diff --git a/src/couchdb/couch_httpd_view.erl b/src/couchdb/couch_httpd_view.erl index a6174a2..da43b6c 100644 --- a/src/couchdb/couch_httpd_view.erl +++ b/src/couchdb/couch_httpd_view.erl @@ -28,7 +28,7 @@ design_doc_view(Req, Db, Id, ViewName, Keys) -> reduce = Reduce } = QueryArgs = parse_view_query(Req, Keys), DesignId = <<"_design/", Id/binary>>, - case couch_view:get_map_view(Db, DesignId, ViewName, Stale) of + Result = case couch_view:get_map_view(Db, DesignId, ViewName, Stale) of {ok, View} -> output_map_view(Req, View, Db, QueryArgs, Keys); {not_found, Reason} -> @@ -45,7 +45,9 @@ design_doc_view(Req, Db, Id, ViewName, Keys) -> _ -> throw({not_found, Reason}) end - end. + end, + couch_stats_collector:increment({httpd, view_reads}), + Result. handle_view_req(#httpd{method='GET',path_parts=[_,_, Id, ViewName]}=Req, Db) -> design_doc_view(Req, Db, Id, ViewName, nil); @@ -60,7 +62,7 @@ handle_view_req(Req, _Db) -> handle_temp_view_req(#httpd{method='POST'}=Req, Db) -> QueryArgs = parse_view_query(Req), - + couch_stats_collector:increment({httpd, temporary_view_reads}), case couch_httpd:primary_header_value(Req, "content-type") of undefined -> ok; "application/json" -> ok; diff --git a/src/couchdb/couch_server.erl b/src/couchdb/couch_server.erl index 69cfa36..f4f0e35 100644 --- a/src/couchdb/couch_server.erl +++ b/src/couchdb/couch_server.erl @@ -182,7 +182,9 @@ maybe_close_lru_db(#server{dbs_open=NumOpen, max_dbs_open=MaxOpen}=Server) maybe_close_lru_db(#server{dbs_open=NumOpen}=Server) -> % must free up the lru db. case try_close_lru(now()) of - ok -> {ok, Server#server{dbs_open=NumOpen-1}}; + ok -> + couch_stats_collector:decrement({couch_db, open_databases}), + {ok, Server#server{dbs_open=NumOpen - 1}}; Error -> Error end. @@ -235,6 +237,7 @@ handle_call({open, DbName, Options}, _From, Server) -> true = ets:insert(couch_dbs_by_pid, {MainPid, DbName}), true = ets:insert(couch_dbs_by_lru, {LruTime, DbName}), DbsOpen = Server2#server.dbs_open + 1, + couch_stats_collector:increment({couch_db, open_databases}), {reply, {ok, MainPid}, Server2#server{dbs_open=DbsOpen}}; Error -> @@ -270,6 +273,7 @@ handle_call({create, DbName, Options}, _From, Server) -> true = ets:insert(couch_dbs_by_pid, {MainPid, DbName}), true = ets:insert(couch_dbs_by_lru, {LruTime, DbName}), DbsOpen = Server2#server.dbs_open + 1, + couch_stats_collector:increment({couch_db, open_databases}), couch_db_update_notifier:notify({created, DbName}), {reply, {ok, MainPid}, Server2#server{dbs_open=DbsOpen}}; @@ -299,6 +303,7 @@ handle_call({delete, DbName, _Options}, _From, Server) -> true = ets:delete(couch_dbs_by_name, DbName), true = ets:delete(couch_dbs_by_pid, Pid), true = ets:delete(couch_dbs_by_lru, LruTime), + couch_stats_collector:decrement({couch_db, open_databases}), Server#server{dbs_open=Server#server.dbs_open - 1} end, case file:delete(FullFilepath) of @@ -328,6 +333,7 @@ handle_info({'EXIT', Pid, _Reason}, #server{dbs_open=DbsOpen}=Server) -> true = ets:delete(couch_dbs_by_pid, Pid), true = ets:delete(couch_dbs_by_name, DbName), true = ets:delete(couch_dbs_by_lru, LruTime), - {noreply, Server#server{dbs_open=DbsOpen-1}}; + couch_stats_collector:decrement({couch_db, open_databases}), + {noreply, Server#server{dbs_open=DbsOpen - 1}}; handle_info(Info, _Server) -> exit({unknown_message, Info}). diff --git a/src/couchdb/couch_server_sup.erl b/src/couchdb/couch_server_sup.erl index 627c34a..57a848e 100644 --- a/src/couchdb/couch_server_sup.erl +++ b/src/couchdb/couch_server_sup.erl @@ -32,6 +32,7 @@ start_link(IniFiles) -> end. restart_core_server() -> + catch couch_stats_collector:stop(), % TODO: move to more appropriate place supervisor:terminate_child(couch_primary_services, couch_server), supervisor:restart_child(couch_primary_services, couch_server). diff --git a/src/couchdb/couch_stats_aggregator.erl b/src/couchdb/couch_stats_aggregator.erl new file mode 100644 index 0000000..f1a17ff --- /dev/null +++ b/src/couchdb/couch_stats_aggregator.erl @@ -0,0 +1,189 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_stats_aggregator). + +-define(TEST, true). +-ifdef(TEST). + -include_lib("eunit/include/eunit.hrl"). +-endif. + +-behaviour(gen_server). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + + +-export([start/0, stop/0, get/1, get/2, time_passed/1]). + +-record(state, {}). + +-define(COLLECTOR, couch_stats_collector). + +% PUBLIC API + +start() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +stop() -> + gen_server:call(?MODULE, stop). + +get(Key) -> + get(Key, []). +get(Key, Options) -> + gen_server:call(?MODULE, {get, Key, Options}). + +time_passed(Time) -> + gen_server:call(?MODULE, {time_passed, Time}). + + +% GEN_SERVER + +init(_) -> + ets:new(?MODULE, [named_table, set, protected]), + lists:map(fun(Time) -> init_counter(Time) end, [1, 5, 15]), + {ok, #state{}}. + +handle_call({get, {Module, Key}, Options}, _, State) -> + Value = + case a2b(Key) of + <<"average_",CollectorKey/binary>> -> + get_average(Module, CollectorKey, Options); + _ -> + ?COLLECTOR:get({b2a(Module), b2a(Key)}) + end, + + {reply, integer_to_binary(Value), State}; + +handle_call(stop, _, State) -> + {stop, normal, stopped, State}; + +% update all counters that match `Time` = int() +handle_call({time_passed, Time}, _, State) -> + lists:foreach(fun(Counter) -> + {{Module, Key, _}, {_, PreviousCount}} = Counter, + CurrentCount = ?COLLECTOR:get({Module, get_collector_key(Key)}), + ets:insert(?MODULE, {{Module, Key, Time}, {PreviousCount, CurrentCount}}) + end, ets:tab2list(?MODULE)), + {reply, ok, State}. + +% PRIVATE API + +get_average_counters() -> + [{httpd, <<"previous_request_count">>}]. + +get_average(Module, Key, Options) -> + Time = proplists:get_value("timeframe", Options, 1) * 60, % default to 1 minute, in seconds + case ets:lookup(?MODULE, {Module, <<"previous_",Key/binary>>, Time}) of + [] -> 0; + [{_, {PreviousCounter, CurrentCounter}}] -> + round((CurrentCounter - PreviousCounter) / Time) + end. + +get_collector_key(Key) -> + <<"previous_", CollectorKey/binary>> = a2b(Key), + b2a(CollectorKey). + +init_counter(Time) -> + Seconds = Time * 60, + lists:map( + fun({Module, Key}) -> + start_timer(Seconds, fun() -> ?MODULE:time_passed(Seconds) end), + ets:insert(?MODULE, {{Module, Key, Time}, {0, 0}}) + end, get_average_counters()). + + +start_timer(Time, Fun) -> + spawn(fun() -> timer(Time * 1000, Fun) end). + +timer(Time, Fun) -> + receive + cancel -> void + after + Time -> Fun(), + timer(Time, Fun) + end. + +% UTILS + +integer_to_binary(Integer) -> + list_to_binary(integer_to_list(Integer)). + +b2a(Binary) when is_atom(Binary)-> + Binary; +b2a(Binary) -> + list_to_atom(binary_to_list(Binary)). + +a2b(Atom) when is_binary(Atom) -> + Atom; +a2b(Atom) -> + list_to_binary(atom_to_list(Atom)). + +% Unused gen_server behaviour API functions that we need to declare. + +%% @doc Unused +handle_cast(foo, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +%% @doc Unused +terminate(_Reason, _State) -> ok. + +%% @doc Unused +code_change(_OldVersion, State, _Extra) -> {ok, State}. + +% TESTS + +test_helper(Fun) -> + catch ?MODULE:stop(), + ?MODULE:start(), + ?COLLECTOR:start(), + + Fun(), + + ?MODULE:stop(), + ?COLLECTOR:stop(). + +should_return_value_from_collector_test() -> + test_helper(fun() -> + ?assertEqual(<<"0">>, ?MODULE:get({couch_db, open_databases})) + end). + +should_handle_multiple_key_value_pairs_test() -> + test_helper(fun() -> + ?COLLECTOR:increment({couch_db, open_databases}), + ?assertEqual(<<"1">>, ?MODULE:get({couch_db, open_databases})), + ?assertEqual(<<"0">>, ?MODULE:get({couch_db, request_count})) + end). + +should_return_the_average_over_the_last_minute_test() -> + test_helper(fun() -> + lists:map(fun(_) -> + ?COLLECTOR:increment({httpd, request_count}) + end, lists:seq(1, 200)), + + ?MODULE:time_passed(60), % seconds + ?assertEqual(<<"3">>, ?MODULE:get({httpd, average_request_count})) + end). + +should_return_the_average_over_the_last_five_minutes_test() -> + test_helper(fun() -> + lists:map(fun(_) -> + ?COLLECTOR:increment({httpd, request_count}) + end, lists:seq(1, 2000)), + + ?MODULE:time_passed(300), % seconds + Result = ?MODULE:get({httpd, average_request_count}, [{"timeframe", list_to_integer("5")}]), + ?assertEqual(<<"7">>, Result) + end). diff --git a/src/couchdb/couch_stats_collector.erl b/src/couchdb/couch_stats_collector.erl new file mode 100644 index 0000000..13fb168 --- /dev/null +++ b/src/couchdb/couch_stats_collector.erl @@ -0,0 +1,155 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_stats_collector). + +-define(TEST, true). +-ifdef(TEST). + -include_lib("eunit/include/eunit.hrl"). +-endif. + +-behaviour(gen_server). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + + +-export([start/0, stop/0, get/1, increment/1, decrement/1]). + +-record(state, {}). + + +% PUBLIC API + +start() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +stop() -> + gen_server:call(?MODULE, stop). + +get(Key) -> + gen_server:call(?MODULE, {get, Key}). + +increment({Module, Key}) when is_integer(Key) -> + gen_server:call(?MODULE, {increment, {Module, list_to_atom(integer_to_list(Key))}}); +increment(Key) -> + gen_server:call(?MODULE, {increment, Key}). + +decrement(Key) -> + gen_server:call(?MODULE, {decrement, Key}). + +% GEN_SERVER + +init(_) -> + ets:new(?MODULE, [named_table, set, protected]), + {ok, #state{}}. + +handle_call({get, Key}, _, State) -> + Result = case ets:lookup(?MODULE, Key) of + [] -> 0; + [{_,Result1}] -> Result1 + end, + {reply, Result, State}; + +handle_call({increment, Key}, _, State) -> + case catch ets:update_counter(?MODULE, Key, 1) of + {'EXIT', {badarg, _}} -> ets:insert(?MODULE, {Key, 1}); + _ -> ok + end, + {reply, ok, State}; + +handle_call({decrement, Key}, _, State) -> + case catch ets:update_counter(?MODULE, Key, -1) of + {'EXIT', {badarg, _}} -> ets:insert(?MODULE, {Key, -1}); + _ -> ok + end, + {reply, ok, State}; + +% handle_call(reset, _, State) -> +% ets:insert(?MODULE, {Key, 0}), +% {reply, ok, 0}; + +handle_call(stop, _, State) -> + {stop, normal, stopped, State}. + + +% Unused gen_server behaviour API functions that we need to declare. + +%% @doc Unused +handle_cast(foo, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +%% @doc Unused +terminate(_Reason, _State) -> ok. + +%% @doc Unused +code_change(_OldVersion, State, _Extra) -> {ok, State}. + +% TESTS + +test_helper(Fun) -> + catch ?MODULE:stop(), + ?MODULE:start(), + + Fun(), + + ?MODULE:stop(). + +should_return_value_from_store_test() -> + test_helper(fun() -> + ?assertEqual(0, ?MODULE:get({couch_db, open_databases})) + end). + +should_increment_value_test() -> + test_helper(fun() -> + ?assert(?MODULE:increment({couch_db, open_databases}) =:= ok), + ?assertEqual(1, ?MODULE:get({couch_db, open_databases})) + end). + +should_decrement_value_test() -> + test_helper(fun() -> + ?assert(?MODULE:decrement({couch_db, open_databases}) =:= ok), + ?assertEqual(-1, ?MODULE:get({couch_db, open_databases})) + end). + +should_increment_and_decrement_value_test() -> + test_helper(fun() -> + ?assert(?MODULE:increment({couch_db, open_databases}) =:= ok), + ?assert(?MODULE:decrement({couch_db, open_databases}) =:= ok), + ?assertEqual(0, ?MODULE:get({couch_db, open_databases})) + end). + +should_reset_counter_value_test() -> + test_helper(fun() -> + ?assert(?MODULE:increment({couch_db, open_databases}) =:= ok), + ?MODULE:stop(), + ?MODULE:start(), + ?assertEqual(0, ?MODULE:get({couch_db, open_databases})) + end). + +should_handle_multiple_key_value_pairs_test() -> + test_helper(fun() -> + ?MODULE:increment({couch_db, open_databases}), + ?assertEqual(1, ?MODULE:get({couch_db, open_databases})), + ?assertEqual(0, ?MODULE:get({couch_db, request_count})) + end). + +should_restart_module_should_create_new_pid_test() -> + test_helper(fun() -> + OldPid = whereis(?MODULE), + ?MODULE:stop(), + ?MODULE:start(), + ?assertNot(whereis(?MODULE) =:= OldPid) + end).