<commit_before>/*
 * Copyright (c) 2012-2017 Daniele Bartolini and individual contributors.
 * License: https://github.com/dbartolini/crown/blob/master/LICENSE
 */

#include "core/containers/vector.h"
#include "core/error/error.h"
#include "core/memory/temp_allocator.h"
#include "core/os.h"
#include "core/platform.h"
#include "core/strings/dynamic_string.h"
#include "core/strings/string_stream.h"
#include <string.h> // strcmp

#if CROWN_PLATFORM_POSIX
	#include <dirent.h> // opendir, readdir
	#include <dlfcn.h>    // dlopen, dlclose, dlsym
	#include <errno.h>
	#include <stdio.h>    // fputs
	#include <stdlib.h>   // getenv
	#include <string.h>   // memset
	#include <sys/stat.h> // stat, mkdir
	#include <sys/wait.h> // wait
	#include <time.h>     // clock_gettime
	#include <unistd.h>   // unlink, rmdir, getcwd, fork, execv
#elif CROWN_PLATFORM_WINDOWS
	#include <io.h>
	#include <stdio.h>
	#include <windows.h>
#endif
#if CROWN_PLATFORM_ANDROID
	#include <android/log.h>
#endif

namespace crown
{
namespace os
{
	s64 clocktime()
	{
#if CROWN_PLATFORM_LINUX || CROWN_PLATFORM_ANDROID
		timespec now;
		clock_gettime(CLOCK_MONOTONIC, &now);
		return now.tv_sec * s64(1000000000) + now.tv_nsec;
#elif CROWN_PLATFORM_OSX
		struct timeval now;
		gettimeofday(&now, NULL);
		return now.tv_sec * s64(1000000) + now.tv_usec;
#elif CROWN_PLATFORM_WINDOWS
		LARGE_INTEGER ttime;
		QueryPerformanceCounter(&ttime);
		return (s64)ttime.QuadPart;
#endif
	}

	s64 clockfrequency()
	{
#if CROWN_PLATFORM_LINUX || CROWN_PLATFORM_ANDROID
		return s64(1000000000);
#elif CROWN_PLATFORM_OSX
		return s64(1000000);
#elif CROWN_PLATFORM_WINDOWS
		LARGE_INTEGER freq;
		QueryPerformanceFrequency(&freq);
		return (s64)freq.QuadPart;
#endif
	}

	/// Suspends execution for @a ms milliseconds.
	void sleep(u32 ms)
	{
#if CROWN_PLATFORM_POSIX
		usleep(ms * 1000);
#elif CROWN_PLATFORM_WINDOWS
		Sleep(ms);
#endif
	}

	/// Opens the library at @a path.
	void* library_open(const char* path)
	{
#if CROWN_PLATFORM_POSIX
		return ::dlopen(path, RTLD_LAZY);
#elif CROWN_PLATFORM_WINDOWS
		return (void*)LoadLibraryA(path);
#endif
	}

	/// Closes a @a library previously opened by library_open.
	void library_close(void* library)
	{
#if CROWN_PLATFORM_POSIX
		dlclose(library);
#elif CROWN_PLATFORM_WINDOWS
		FreeLibrary((HMODULE)library);
#endif
	}

	/// Returns a pointer to the symbol @a name in the given @a library.
	void* library_symbol(void* library, const char* name)
	{
#if CROWN_PLATFORM_POSIX
		return ::dlsym(library, name);
#elif CROWN_PLATFORM_WINDOWS
		return (void*)GetProcAddress((HMODULE)library, name);
#endif
	}

	/// Logs the message @a msg.
	void log(const char* msg)
	{
#if CROWN_PLATFORM_ANDROID
		__android_log_write(ANDROID_LOG_DEBUG, "crown", msg);
#elif CROWN_PLATFORM_WINDOWS
		OutputDebugStringA(msg);
#else
		fputs(msg, stdout);
		fflush(stdout);
#endif
	}

	void stat(Stat& info, const char* path)
	{
		info.file_type = Stat::NO_ENTRY;
		info.size = 0;
		info.mtime = 0;

#if CROWN_PLATFORM_POSIX
		struct stat buf;
		memset(&buf, 0, sizeof(buf));
		int err = ::stat(path, &buf);
		if (err != 0)
			return;

		if (S_ISREG(buf.st_mode) == 1)
			info.file_type = Stat::REGULAR;
		else if (S_ISDIR(buf.st_mode) == 1)
			info.file_type = Stat::DIRECTORY;
#elif CROWN_PLATFORM_WINDOWS
		struct _stat64 buf;
		int err = ::_stat64(path, &buf);
		if (err != 0)
			return;

		if ((buf.st_mode & _S_IFREG) != 0)
			info.file_type = Stat::REGULAR;
		else if ((buf.st_mode & _S_IFDIR) != 0)
			info.file_type = Stat::DIRECTORY;
#endif

		info.size = buf.st_size;
		info.mtime = buf.st_mtime;
	}

	/// Deletes the file at @a path.
	void delete_file(const char* path)
	{
#if CROWN_PLATFORM_POSIX
		int err = ::unlink(path);
		CE_ASSERT(err == 0, "unlink: errno = %d", errno);
		CE_UNUSED(err);
#elif CROWN_PLATFORM_WINDOWS
		BOOL err = DeleteFile(path);
		CE_ASSERT(err != 0, "DeleteFile: GetLastError = %d", GetLastError());
		CE_UNUSED(err);
#endif
	}

	/// Creates a directory named @a path.
	void create_directory(const char* path)
	{
#if CROWN_PLATFORM_POSIX
		int err = ::mkdir(path, 0755);
		CE_ASSERT(err == 0, "mkdir: errno = %d", errno);
		CE_UNUSED(err);
#elif CROWN_PLATFORM_WINDOWS
		BOOL err = CreateDirectory(path, NULL);
		CE_ASSERT(err != 0, "CreateDirectory: GetLastError = %d", GetLastError());
		CE_UNUSED(err);
#endif
	}

	/// Deletes the directory at @a path.
	void delete_directory(const char* path)
	{
#if CROWN_PLATFORM_POSIX
		int err = ::rmdir(path);
		CE_ASSERT(err == 0, "rmdir: errno = %d", errno);
		CE_UNUSED(err);
#elif CROWN_PLATFORM_WINDOWS
		BOOL err = RemoveDirectory(path);
		CE_ASSERT(err != 0, "RemoveDirectory: GetLastError = %d", GetLastError());
		CE_UNUSED(err);
#endif
	}

	/// Returns the list of @a files at the given @a path.
	void list_files(const char* path, Vector<DynamicString>& files);

	/// Returns the current working directory.
	const char* getcwd(char* buf, u32 size)
	{
#if CROWN_PLATFORM_POSIX
		return ::getcwd(buf, size);
#elif CROWN_PLATFORM_WINDOWS
		GetCurrentDirectory(size, buf);
		return buf;
#endif
	}

	/// Returns the value of the environment variable @a name.
	const char* getenv(const char* name)
	{
#if CROWN_PLATFORM_POSIX
		return ::getenv(name);
#elif CROWN_PLATFORM_WINDOWS
		// GetEnvironmentVariable(name, buf, size);
		return NULL;
#endif
	}

	void list_files(const char* path, Vector<DynamicString>& files)
	{
#if CROWN_PLATFORM_POSIX
		DIR *dir;
		struct dirent *entry;

		if (!(dir = opendir(path)))
			return;

		while ((entry = readdir(dir)))
		{
			const char* dname = entry->d_name;

			if (!strcmp(dname, ".") || !strcmp(dname, ".."))
				continue;

			TempAllocator512 ta;
			DynamicString fname(ta);
			fname.set(dname, strlen32(dname));
			vector::push_back(files, fname);
		}

		closedir(dir);
#elif CROWN_PLATFORM_WINDOWS
		TempAllocator1024 ta;
		DynamicString cur_path(ta);
		cur_path += path;
		cur_path += "\\*";

		WIN32_FIND_DATA ffd;
		HANDLE file = FindFirstFile(cur_path.c_str(), &ffd);
		if (file == INVALID_HANDLE_VALUE)
			return;

		do
		{
			const char* dname = ffd.cFileName;

			if (!strcmp(dname, ".") || !strcmp(dname, ".."))
				continue;

			TempAllocator512 ta;
			DynamicString fname(ta);
			fname.set(dname, strlen32(dname));
			vector::push_back(files, fname);
		}
		while (FindNextFile(file, &ffd) != 0);

		FindClose(file);
#endif
	}

	int execute_process(const char* const* argv, StringStream& output)
	{
		TempAllocator512 ta;
		StringStream path(ta);

		path << argv[0];
		path << ' ';
#if CROWN_PLATFORM_POSIX
		path << "2>&1 ";
#endif
		for (s32 i = 1; argv[i] != NULL; ++i)
		{
			const char* arg = argv[i];
			for (; *arg; ++arg)
			{
				if (*arg == ' ')
					path << '\\';
				path << *arg;
			}
			path << ' ';
		}
#if CROWN_PLATFORM_POSIX
		FILE* file = popen(string_stream::c_str(path), "r");

		char buf[1024];
		while (fgets(buf, sizeof(buf), file) != NULL)
			output << buf;

		return pclose(file);
#elif CROWN_PLATFORM_WINDOWS
		STARTUPINFO info;
		memset(&info, 0, sizeof(info));
		info.cb = sizeof(info);

		PROCESS_INFORMATION process;
		memset(&process, 0, sizeof(process));

		int err = CreateProcess(argv[0]
			, (LPSTR)string_stream::c_str(path)
			, NULL
			, NULL
			, TRUE
			, 0
			, NULL
			, NULL
			, &info
			, &process
			);
		CE_ASSERT(err != 0, "CreateProcess: GetLastError = %d", GetLastError());
		CE_UNUSED(err);

		DWORD exitcode = 1;
		::WaitForSingleObject(process.hProcess, INFINITE);
		GetExitCodeProcess(process.hProcess, &exitcode);
		CloseHandle(process.hProcess);
		CloseHandle(process.hThread);
		return (int)exitcode;
#endif
	}

} // namespace os

} // namespace crown
<commit_msg>Fix build<commit_after>/*
 * Copyright (c) 2012-2017 Daniele Bartolini and individual contributors.
 * License: https://github.com/dbartolini/crown/blob/master/LICENSE
 */

#include "core/containers/vector.h"
#include "core/error/error.h"
#include "core/memory/temp_allocator.h"
#include "core/os.h"
#include "core/platform.h"
#include "core/strings/dynamic_string.h"
#include "core/strings/string_stream.h"
#include <string.h>   // strcmp
#include <sys/stat.h> // stat, mkdir

#if CROWN_PLATFORM_POSIX
	#include <dirent.h> // opendir, readdir
	#include <dlfcn.h>    // dlopen, dlclose, dlsym
	#include <errno.h>
	#include <stdio.h>    // fputs
	#include <stdlib.h>   // getenv
	#include <string.h>   // memset
	#include <sys/wait.h> // wait
	#include <time.h>     // clock_gettime
	#include <unistd.h>   // unlink, rmdir, getcwd, fork, execv
#elif CROWN_PLATFORM_WINDOWS
	#include <io.h>
	#include <stdio.h>
	#include <windows.h>
#endif
#if CROWN_PLATFORM_ANDROID
	#include <android/log.h>
#endif

namespace crown
{
namespace os
{
	s64 clocktime()
	{
#if CROWN_PLATFORM_LINUX || CROWN_PLATFORM_ANDROID
		timespec now;
		clock_gettime(CLOCK_MONOTONIC, &now);
		return now.tv_sec * s64(1000000000) + now.tv_nsec;
#elif CROWN_PLATFORM_OSX
		struct timeval now;
		gettimeofday(&now, NULL);
		return now.tv_sec * s64(1000000) + now.tv_usec;
#elif CROWN_PLATFORM_WINDOWS
		LARGE_INTEGER ttime;
		QueryPerformanceCounter(&ttime);
		return (s64)ttime.QuadPart;
#endif
	}

	s64 clockfrequency()
	{
#if CROWN_PLATFORM_LINUX || CROWN_PLATFORM_ANDROID
		return s64(1000000000);
#elif CROWN_PLATFORM_OSX
		return s64(1000000);
#elif CROWN_PLATFORM_WINDOWS
		LARGE_INTEGER freq;
		QueryPerformanceFrequency(&freq);
		return (s64)freq.QuadPart;
#endif
	}

	/// Suspends execution for @a ms milliseconds.
	void sleep(u32 ms)
	{
#if CROWN_PLATFORM_POSIX
		usleep(ms * 1000);
#elif CROWN_PLATFORM_WINDOWS
		Sleep(ms);
#endif
	}

	/// Opens the library at @a path.
	void* library_open(const char* path)
	{
#if CROWN_PLATFORM_POSIX
		return ::dlopen(path, RTLD_LAZY);
#elif CROWN_PLATFORM_WINDOWS
		return (void*)LoadLibraryA(path);
#endif
	}

	/// Closes a @a library previously opened by library_open.
	void library_close(void* library)
	{
#if CROWN_PLATFORM_POSIX
		dlclose(library);
#elif CROWN_PLATFORM_WINDOWS
		FreeLibrary((HMODULE)library);
#endif
	}

	/// Returns a pointer to the symbol @a name in the given @a library.
	void* library_symbol(void* library, const char* name)
	{
#if CROWN_PLATFORM_POSIX
		return ::dlsym(library, name);
#elif CROWN_PLATFORM_WINDOWS
		return (void*)GetProcAddress((HMODULE)library, name);
#endif
	}

	/// Logs the message @a msg.
	void log(const char* msg)
	{
#if CROWN_PLATFORM_ANDROID
		__android_log_write(ANDROID_LOG_DEBUG, "crown", msg);
#elif CROWN_PLATFORM_WINDOWS
		OutputDebugStringA(msg);
#else
		fputs(msg, stdout);
		fflush(stdout);
#endif
	}

	void stat(Stat& info, const char* path)
	{
		info.file_type = Stat::NO_ENTRY;
		info.size = 0;
		info.mtime = 0;

#if CROWN_PLATFORM_POSIX
		struct stat buf;
		memset(&buf, 0, sizeof(buf));
		int err = ::stat(path, &buf);
		if (err != 0)
			return;

		if (S_ISREG(buf.st_mode) == 1)
			info.file_type = Stat::REGULAR;
		else if (S_ISDIR(buf.st_mode) == 1)
			info.file_type = Stat::DIRECTORY;
#elif CROWN_PLATFORM_WINDOWS
		struct _stat64 buf;
		int err = ::_stat64(path, &buf);
		if (err != 0)
			return;

		if ((buf.st_mode & _S_IFREG) != 0)
			info.file_type = Stat::REGULAR;
		else if ((buf.st_mode & _S_IFDIR) != 0)
			info.file_type = Stat::DIRECTORY;
#endif

		info.size = buf.st_size;
		info.mtime = buf.st_mtime;
	}

	/// Deletes the file at @a path.
	void delete_file(const char* path)
	{
#if CROWN_PLATFORM_POSIX
		int err = ::unlink(path);
		CE_ASSERT(err == 0, "unlink: errno = %d", errno);
		CE_UNUSED(err);
#elif CROWN_PLATFORM_WINDOWS
		BOOL err = DeleteFile(path);
		CE_ASSERT(err != 0, "DeleteFile: GetLastError = %d", GetLastError());
		CE_UNUSED(err);
#endif
	}

	/// Creates a directory named @a path.
	void create_directory(const char* path)
	{
#if CROWN_PLATFORM_POSIX
		int err = ::mkdir(path, 0755);
		CE_ASSERT(err == 0, "mkdir: errno = %d", errno);
		CE_UNUSED(err);
#elif CROWN_PLATFORM_WINDOWS
		BOOL err = CreateDirectory(path, NULL);
		CE_ASSERT(err != 0, "CreateDirectory: GetLastError = %d", GetLastError());
		CE_UNUSED(err);
#endif
	}

	/// Deletes the directory at @a path.
	void delete_directory(const char* path)
	{
#if CROWN_PLATFORM_POSIX
		int err = ::rmdir(path);
		CE_ASSERT(err == 0, "rmdir: errno = %d", errno);
		CE_UNUSED(err);
#elif CROWN_PLATFORM_WINDOWS
		BOOL err = RemoveDirectory(path);
		CE_ASSERT(err != 0, "RemoveDirectory: GetLastError = %d", GetLastError());
		CE_UNUSED(err);
#endif
	}

	/// Returns the list of @a files at the given @a path.
	void list_files(const char* path, Vector<DynamicString>& files);

	/// Returns the current working directory.
	const char* getcwd(char* buf, u32 size)
	{
#if CROWN_PLATFORM_POSIX
		return ::getcwd(buf, size);
#elif CROWN_PLATFORM_WINDOWS
		GetCurrentDirectory(size, buf);
		return buf;
#endif
	}

	/// Returns the value of the environment variable @a name.
	const char* getenv(const char* name)
	{
#if CROWN_PLATFORM_POSIX
		return ::getenv(name);
#elif CROWN_PLATFORM_WINDOWS
		// GetEnvironmentVariable(name, buf, size);
		return NULL;
#endif
	}

	void list_files(const char* path, Vector<DynamicString>& files)
	{
#if CROWN_PLATFORM_POSIX
		DIR *dir;
		struct dirent *entry;

		if (!(dir = opendir(path)))
			return;

		while ((entry = readdir(dir)))
		{
			const char* dname = entry->d_name;

			if (!strcmp(dname, ".") || !strcmp(dname, ".."))
				continue;

			TempAllocator512 ta;
			DynamicString fname(ta);
			fname.set(dname, strlen32(dname));
			vector::push_back(files, fname);
		}

		closedir(dir);
#elif CROWN_PLATFORM_WINDOWS
		TempAllocator1024 ta;
		DynamicString cur_path(ta);
		cur_path += path;
		cur_path += "\\*";

		WIN32_FIND_DATA ffd;
		HANDLE file = FindFirstFile(cur_path.c_str(), &ffd);
		if (file == INVALID_HANDLE_VALUE)
			return;

		do
		{
			const char* dname = ffd.cFileName;

			if (!strcmp(dname, ".") || !strcmp(dname, ".."))
				continue;

			TempAllocator512 ta;
			DynamicString fname(ta);
			fname.set(dname, strlen32(dname));
			vector::push_back(files, fname);
		}
		while (FindNextFile(file, &ffd) != 0);

		FindClose(file);
#endif
	}

	int execute_process(const char* const* argv, StringStream& output)
	{
		TempAllocator512 ta;
		StringStream path(ta);

		path << argv[0];
		path << ' ';
#if CROWN_PLATFORM_POSIX
		path << "2>&1 ";
#endif
		for (s32 i = 1; argv[i] != NULL; ++i)
		{
			const char* arg = argv[i];
			for (; *arg; ++arg)
			{
				if (*arg == ' ')
					path << '\\';
				path << *arg;
			}
			path << ' ';
		}
#if CROWN_PLATFORM_POSIX
		FILE* file = popen(string_stream::c_str(path), "r");

		char buf[1024];
		while (fgets(buf, sizeof(buf), file) != NULL)
			output << buf;

		return pclose(file);
#elif CROWN_PLATFORM_WINDOWS
		STARTUPINFO info;
		memset(&info, 0, sizeof(info));
		info.cb = sizeof(info);

		PROCESS_INFORMATION process;
		memset(&process, 0, sizeof(process));

		int err = CreateProcess(argv[0]
			, (LPSTR)string_stream::c_str(path)
			, NULL
			, NULL
			, TRUE
			, 0
			, NULL
			, NULL
			, &info
			, &process
			);
		CE_ASSERT(err != 0, "CreateProcess: GetLastError = %d", GetLastError());
		CE_UNUSED(err);

		DWORD exitcode = 1;
		::WaitForSingleObject(process.hProcess, INFINITE);
		GetExitCodeProcess(process.hProcess, &exitcode);
		CloseHandle(process.hProcess);
		CloseHandle(process.hThread);
		return (int)exitcode;
#endif
	}

} // namespace os

} // namespace crown
<|endoftext|>


<commit_before>/*******************************************************************************
  Copyright (c) 2011 Dmitry Matveev <me@dmitrymatveev.co.uk>

  Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files (the "Software"), to deal
  in the Software without restriction, including without limitation the rights
  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  copies of the Software, and to permit persons to whom the Software is
  furnished to do so, subject to the following conditions:

  The above copyright notice and this permission notice shall be included in
  all copies or substantial portions of the Software.

  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  THE SOFTWARE.
*******************************************************************************/

#include <fcntl.h>
#include <unistd.h>

#include <cstdlib>

#include "open_close_test.hh"

open_close_test::open_close_test (journal &j)
: test ("Open/close notifications", j)
{
}

void open_close_test::setup ()
{
    cleanup ();
    system ("touch oct-file-working");
    system ("echo Hello >> oct-file-working");
    system ("mkdir oct-dir-working");
}

void open_close_test::run ()
{
    consumer cons;
    int file_wid = 0;
    int dir_wid = 0;
    int fd, dup_fd;
    events received;

    cons.input.setup ("oct-file-working", IN_OPEN | IN_CLOSE_WRITE | IN_CLOSE_NOWRITE);
    cons.output.wait ();

    file_wid = cons.output.added_watch_id ();
    should ("start watching on a file", file_wid != -1);

    cons.output.reset ();
    cons.input.setup ("oct-dir-working", IN_OPEN | IN_CLOSE_WRITE | IN_CLOSE_NOWRITE);
    cons.output.wait ();

    dir_wid = cons.output.added_watch_id ();
    should ("start watching on a directory", dir_wid != -1);

    cons.output.reset ();
    cons.input.receive ();

    system ("cat oct-file-working >> /dev/null");

    cons.output.wait ();
    received = cons.output.registered ();
    should ("receive IN_OPEN on cat",
            contains (received, event ("", file_wid, IN_OPEN)));
    should ("receive IN_CLOSE_NOWRITE on cat",
            contains (received, event ("", file_wid, IN_CLOSE_NOWRITE)));

    cons.output.reset ();
    cons.input.receive ();

    system ("ls oct-dir-working >> /dev/null");

    cons.output.wait ();
    received = cons.output.registered ();
    should ("receive IN_OPEN on ls",
            contains (received, event ("", dir_wid, IN_OPEN)));
    should ("receive IN_CLOSE_NOWRITE on ls",
            contains (received, event ("", dir_wid, IN_CLOSE_NOWRITE)));

    cons.output.reset ();
    cons.input.receive ();

    system ("echo Hello >> oct-file-working");

    cons.output.wait ();
    received = cons.output.registered ();
    should ("receive IN_OPEN on modify",
            contains (received, event ("", file_wid, IN_OPEN)));
    should ("receive IN_CLOSE_WRITE on modify",
            contains (received, event ("", file_wid, IN_CLOSE_WRITE)));

    cons.output.reset ();
    cons.input.receive ();

    fd = open ("oct-file-working", O_RDONLY);
    dup_fd = dup (fd);
    close (fd);

    cons.output.wait ();
    received = cons.output.registered ();
    should ("not receive IN_CLOSE_NOWRITE on close() "
            "of non last dup()-ed fd opened for read",
            !contains (received, event ("", file_wid, IN_CLOSE_NOWRITE)));

    cons.output.reset ();
    cons.input.receive ();

    close (dup_fd);

    cons.output.wait ();
    received = cons.output.registered ();
    should ("receive IN_CLOSE_NOWRITE on close() "
            "of last dup()-ed fd opened for read",
            contains (received, event ("", file_wid, IN_CLOSE_NOWRITE)));

    cons.output.reset ();
    cons.input.receive ();

    fd = open ("oct-file-working", O_WRONLY);
    dup_fd = dup (fd);
    close (fd);

    cons.output.wait ();
    received = cons.output.registered ();
    should ("not receive IN_CLOSE_WRITE on close() "
            "of non last dup()-ed fd opened for write",
            !contains (received, event ("", file_wid, IN_CLOSE_WRITE)));

    cons.output.reset ();
    cons.input.receive ();

    close (dup_fd);

    cons.output.wait ();
    received = cons.output.registered ();
    should ("receive IN_CLOSE_WRITE on close() "
            "of last dup()-ed fd opened for write",
            contains (received, event ("", file_wid, IN_CLOSE_WRITE)));

    cons.input.interrupt ();
}

void open_close_test::cleanup ()
{
    system ("rm -rf oct-file-working");
    system ("rm -rf oct-dir-working");
}
<commit_msg>Revert "Unit tests for closing of duped filedescs"<commit_after>/*******************************************************************************
  Copyright (c) 2011 Dmitry Matveev <me@dmitrymatveev.co.uk>

  Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files (the "Software"), to deal
  in the Software without restriction, including without limitation the rights
  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  copies of the Software, and to permit persons to whom the Software is
  furnished to do so, subject to the following conditions:

  The above copyright notice and this permission notice shall be included in
  all copies or substantial portions of the Software.

  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  THE SOFTWARE.
*******************************************************************************/

#include <cstdlib>

#include "open_close_test.hh"

open_close_test::open_close_test (journal &j)
: test ("Open/close notifications", j)
{
}

void open_close_test::setup ()
{
    cleanup ();
    system ("touch oct-file-working");
    system ("echo Hello >> oct-file-working");
    system ("mkdir oct-dir-working");
}

void open_close_test::run ()
{
    consumer cons;
    int file_wid = 0;
    int dir_wid = 0;
    events received;

    cons.input.setup ("oct-file-working", IN_OPEN | IN_CLOSE_WRITE | IN_CLOSE_NOWRITE);
    cons.output.wait ();

    file_wid = cons.output.added_watch_id ();
    should ("start watching on a file", file_wid != -1);

    cons.output.reset ();
    cons.input.setup ("oct-dir-working", IN_OPEN | IN_CLOSE_WRITE | IN_CLOSE_NOWRITE);
    cons.output.wait ();

    dir_wid = cons.output.added_watch_id ();
    should ("start watching on a directory", dir_wid != -1);

    cons.output.reset ();
    cons.input.receive ();

    system ("cat oct-file-working >> /dev/null");

    cons.output.wait ();
    received = cons.output.registered ();
    should ("receive IN_OPEN on cat",
            contains (received, event ("", file_wid, IN_OPEN)));
    should ("receive IN_CLOSE_NOWRITE on cat",
            contains (received, event ("", file_wid, IN_CLOSE_NOWRITE)));

    cons.output.reset ();
    cons.input.receive ();

    system ("ls oct-dir-working >> /dev/null");

    cons.output.wait ();
    received = cons.output.registered ();
    should ("receive IN_OPEN on ls",
            contains (received, event ("", dir_wid, IN_OPEN)));
    should ("receive IN_CLOSE_NOWRITE on ls",
            contains (received, event ("", dir_wid, IN_CLOSE_NOWRITE)));

    cons.output.reset ();
    cons.input.receive ();

    system ("echo Hello >> oct-file-working");

    cons.output.wait ();
    received = cons.output.registered ();
    should ("receive IN_OPEN on modify",
            contains (received, event ("", file_wid, IN_OPEN)));
    should ("receive IN_CLOSE_WRITE on modify",
            contains (received, event ("", file_wid, IN_CLOSE_WRITE)));

    cons.input.interrupt ();
}

void open_close_test::cleanup ()
{
    system ("rm -rf oct-file-working");
    system ("rm -rf oct-dir-working");
}
<|endoftext|>


<commit_before>// License: Apache 2.0. See LICENSE file in root directory.
// Copyright(c) 2018 Intel Corporation. All Rights Reserved.

#include <iostream>
#include <string>
#include <string.h> // memcpy
#include <fstream>
#include <vector>

#include "tclap/CmdLine.h"

#define STB_IMAGE_IMPLEMENTATION
#include "../third-party/stb_image.h"

#include <lz4.h>

#define RS_EMBED_VERSION "0.0.0.1"

struct float3
{
    float x, y, z;
};

struct short3
{
    uint16_t x, y, z;
};

using namespace std;
using namespace TCLAP;

bool file_exists(const std::string& name) {
    std::ifstream f(name.c_str());
    return f.good();
}

struct int6
{
    int x, y, z, a, b, c;
};

void uncompress(const uint8_t* ptr, int compressed_size, int original_size, int vertex_count, int index_count,
    std::vector<float3>& vertex_data, std::vector<float3>& normals, std::vector<short3>& index_data)
{
    std::vector<char> uncompressed(original_size, 0);
    LZ4_decompress_safe((const char*)ptr, uncompressed.data(), compressed_size, original_size);
    const int vertex_size = vertex_count * sizeof(float3);
    const int index_size = index_count * sizeof(short3);
    vertex_data.resize(vertex_count);
    memcpy(vertex_data.data(), uncompressed.data(), vertex_size);
    index_data.resize(index_count);
    memcpy(index_data.data(), uncompressed.data() + vertex_size, index_size);
    normals.resize(vertex_count);
    memcpy(normals.data(), uncompressed.data() + vertex_size + index_size, vertex_size);
}

bool ends_with(const std::string& s, const std::string& suffix)
{
    auto i = s.rbegin(), j = suffix.rbegin();
    for (; i != s.rend() && j != suffix.rend() && *i == *j;
        i++, j++);
    return j == suffix.rend();
}

int main(int argc, char** argv) try
{
    // Parse command line arguments
    CmdLine cmd("librealsense rs-embed tool", ' ', RS_EMBED_VERSION);
    ValueArg<string> inputFilename("i", "input", "Input filename", true, "", "input-file");
    ValueArg<string> outputFilename("o", "output", "Output filename", false, "", "output-file");
    ValueArg<string> objectName("n", "name", "Name", false, "", "object-name");

    cmd.add(inputFilename);
    cmd.add(outputFilename);
    cmd.add(objectName);
    cmd.parse(argc, argv);

    auto input = inputFilename.getValue();
    auto output = outputFilename.getValue();
    auto name = objectName.getValue();

    if (ends_with(input, ".obj"))
    {
        std::vector<float3> vertex_data;
        std::vector<float3> normals_raw_data;
        std::vector<float3> normals_data;
        std::vector<int6> index_raw_data;
        std::vector<short3> index_data;

        if (file_exists(input))
        {
            std::ifstream file(input);
            std::string str;
            while (std::getline(file, str))
            {
                if (str.size())
                {
                    if (str[0] == 'v')
                    {
                        float a, b, c;
                        if (str[1] == 'n')
                        {
                            sscanf(str.c_str(), "vn %f %f %f", &a, &b, &c);
                            normals_raw_data.push_back({ a, b, c });
                        }
                        else
                        {
                            sscanf(str.c_str(), "v %f %f %f", &a, &b, &c);
                            vertex_data.push_back({ a, b, c });
                        }
                    }
                    if (str[0] == 'f')
                    {
                        int x, y, z, a = 0, b = 0, c = 0; // TODO check about a,b,c values
                        sscanf(str.c_str(), "f %d %d %d", &x, &y, &z);
                        index_raw_data.push_back({ x, y, z, a, b, c });
                    }
                }
            }
        }

        normals_data.resize(vertex_data.size());
        for (auto& idx : index_raw_data)
        {
            // normals_data[idx.x] = normals_raw_data[idx.a];
            // normals_data[idx.y] = normals_raw_data[idx.b];
            // normals_data[idx.z] = normals_raw_data[idx.c];
            index_data.push_back({ static_cast<uint16_t>(idx.x - 1),
                                    static_cast<uint16_t>(idx.y - 1),
                                    static_cast<uint16_t>(idx.z - 1) });
        }

        size_t vertex_data_size = vertex_data.size() * sizeof(float3);
        size_t index_data_size = index_data.size() * sizeof(short3);
        //size_t normals_data_size = normals_data.size() * sizeof(float3);

        std::vector<uint8_t> data(vertex_data_size + index_data_size, 0);
        memcpy(data.data(), vertex_data.data(), vertex_data_size);
        memcpy(data.data() + vertex_data_size, index_data.data(), index_data_size);
        //memcpy(data.data() + vertex_data_size + index_data_size, normals_data.data(), normals_data_size);

        // compress szSource into pchCompressed
        auto rawDataSize = (int)data.size();
        auto compressBufSize = LZ4_compressBound(rawDataSize);
        char* pchCompressed = new char[compressBufSize];
        memset(pchCompressed, compressBufSize, 0);
        int nCompressedSize = LZ4_compress_default((const char*)data.data(), pchCompressed, rawDataSize, compressBufSize);

        ofstream myfile;
        myfile.open(output);
        myfile << "// License: Apache 2.0. See LICENSE file in root directory.\n";
        myfile << "// Copyright(c) 2018 Intel Corporation. All Rights Reserved.\n\n";
        myfile << "// This file is auto-generated from " << name << ".obj\n";
        myfile << "#pragma once\n";
        myfile << "static uint32_t " << name << "_obj_data [] { ";

        auto nAllignedCompressedSize = nCompressedSize;
        auto leftover = nCompressedSize % 4;
        if (leftover % 4 != 0) nAllignedCompressedSize += (4 - leftover);

        for (int i = 0; i < nAllignedCompressedSize; i += 4)
        {
            uint32_t* ptr = (uint32_t*)(pchCompressed + i);
            myfile << "0x" << std::hex << (int)(*ptr);
            if (i < nAllignedCompressedSize - 1) myfile << ",";
        }

        myfile << "};\n";

        myfile << "#include <lz4.h>\n";
        myfile << "#include <vector>\n";

        myfile << "inline void uncompress_" << name << "_obj(std::vector<float3>& vertex_data, std::vector<float3>& normals, std::vector<short3>& index_data)\n";
        myfile << "{\n";
        myfile << "    std::vector<char> uncompressed(0x" << std::hex << rawDataSize << ", 0);\n";
        myfile << "    LZ4_decompress_safe((const char*)" << name << "_obj_data, uncompressed.data(), 0x" << std::hex << nCompressedSize << ", 0x" << std::hex << rawDataSize << ");\n";
        myfile << "    const int vertex_size = 0x" << std::hex << vertex_data.size() << " * sizeof(float3);\n";
        myfile << "    const int index_size = 0x" << std::hex << index_data.size() << " * sizeof(short3);\n";
        myfile << "    vertex_data.resize(0x" << std::hex << vertex_data.size() << ");\n";
        myfile << "    memcpy(vertex_data.data(), uncompressed.data(), vertex_size);\n";
        myfile << "    index_data.resize(0x" << std::hex << index_data.size() << ");\n";
        myfile << "    memcpy(index_data.data(), uncompressed.data() + vertex_size, index_size);\n";
        myfile << "    //normals.resize(0x" << std::hex << vertex_data.size() << ");\n";
        myfile << "    //memcpy(normals.data(), uncompressed.data() + vertex_size + index_size, vertex_size);\n";
        myfile << "}\n";

        myfile.close();
    }

    if (ends_with(input, ".png"))
    {
        ifstream ifs(input, ios::binary | ios::ate);
        ifstream::pos_type pos = ifs.tellg();

        std::vector<char> buffer(pos);

        ifs.seekg(0, ios::beg);
        ifs.read(&buffer[0], pos);

        ofstream myfile;
        myfile.open(output);
        myfile << "// License: Apache 2.0. See LICENSE file in root directory.\n";
        myfile << "// Copyright(c) 2018 Intel Corporation. All Rights Reserved.\n\n";
        myfile << "// This file is auto-generated from " << name << ".png\n";

        myfile << "static uint32_t " << name << "_png_size = 0x" << std::hex << buffer.size() << ";\n";

        myfile << "static uint8_t " << name << "_png_data [] { ";
        for (int i = 0; i < buffer.size(); i++)
        {
            uint8_t byte = buffer[i];
            myfile << "0x" << std::hex << (int)byte;
            if (i < buffer.size() - 1) myfile << ",";
        }
        myfile << "};\n";

        myfile.close();
    }

    //uncompress_d435_obj(vertex_data, normals_data, index_data);

    return EXIT_SUCCESS;
}
catch (const exception& e)
{
    cerr << e.what() << endl;
    return EXIT_FAILURE;
}
catch (...)
{
    cerr << "some error" << endl;
    return EXIT_FAILURE;
}
<commit_msg>support Vertex normal indices without texture coordinate indices format<commit_after>// License: Apache 2.0. See LICENSE file in root directory.
// Copyright(c) 2021 Intel Corporation. All Rights Reserved.

#include <iostream>
#include <string>
#include <string.h> // memcpy
#include <fstream>
#include <vector>

#include "tclap/CmdLine.h"
#include <lz4.h>

#define STB_IMAGE_IMPLEMENTATION
#include "../third-party/stb_image.h"

#define RS_EMBED_VERSION "0.0.0.1"

struct float3
{
    float x, y, z;
};

struct short3
{
    uint16_t x, y, z;
};

using namespace std;
using namespace TCLAP;

bool file_exists(const std::string& name) {
    std::ifstream f(name.c_str());
    return f.good();
}

struct int6
{
    int x, y, z, a, b, c;
};

bool ends_with(const std::string& s, const std::string& suffix)
{
    auto i = s.rbegin(), j = suffix.rbegin();
    for (; i != s.rend() && j != suffix.rend() && *i == *j;
        i++, j++);
    return j == suffix.rend();
}

int main(int argc, char** argv) try
{
    // Parse command line arguments
    CmdLine cmd("librealsense rs-embed tool", ' ', RS_EMBED_VERSION);
    ValueArg<string> inputFilename("i", "input", "Input filename", true, "", "input-file");
    ValueArg<string> outputFilename("o", "output", "Output filename", false, "", "output-file");
    ValueArg<string> objectName("n", "name", "Name", false, "", "object-name");

    cmd.add(inputFilename);
    cmd.add(outputFilename);
    cmd.add(objectName);
    cmd.parse(argc, argv);

    auto input = inputFilename.getValue();
    auto output = outputFilename.getValue();
    auto name = objectName.getValue();

    if (ends_with(input, ".obj"))
    {
        std::vector<float3> vertex_data;
        std::vector<float3> normals_raw_data;
        std::vector<float3> normals_data;
        std::vector<int6> index_raw_data;
        std::vector<short3> index_data;

        if (file_exists(input))
        {
            std::ifstream file(input);
            std::string str;
            while (std::getline(file, str))
            {
                if (str.size())
                {
                    if (str[0] == 'v')
                    {
                        float a, b, c;
                        if (str[1] == 'n')
                        {
                            sscanf(str.c_str(), "vn %f %f %f", &a, &b, &c);
                            normals_raw_data.push_back({ a, b, c });
                        }
                        else
                        {
                            sscanf(str.c_str(), "v %f %f %f", &a, &b, &c);
                            vertex_data.push_back({ a, b, c });
                        }
                    }
                    if (str[0] == 'f')
                    {
                        int x, y, z, a, b, c;
                        sscanf(str.c_str(), "f %d//%d %d//%d %d//%d", &x, &a, &y, &b, &z, &c);
                        index_raw_data.push_back({ x, y, z, a, b, c });
                    }
                }
            }
        }
        else
        {
            std::cout << "file: " << input << " could not be found!" << std::endl;
            return EXIT_FAILURE;
        }

        normals_data.resize(vertex_data.size());
        for (auto& idx : index_raw_data)
        {
            // TODO - normals data are currently disabled
            // normals_data[idx.x] = normals_raw_data[idx.a];
            // normals_data[idx.y] = normals_raw_data[idx.b];
            // normals_data[idx.z] = normals_raw_data[idx.c];
            index_data.push_back({ static_cast<uint16_t>(idx.x - 1),
                                    static_cast<uint16_t>(idx.y - 1),
                                    static_cast<uint16_t>(idx.z - 1) });
        }

        size_t vertex_data_size = vertex_data.size() * sizeof(float3);
        size_t index_data_size = index_data.size() * sizeof(short3);
        //size_t normals_data_size = normals_data.size() * sizeof(float3);

        std::vector<uint8_t> data(vertex_data_size + index_data_size, 0);
        memcpy(data.data(), vertex_data.data(), vertex_data_size);
        memcpy(data.data() + vertex_data_size, index_data.data(), index_data_size);
        //memcpy(data.data() + vertex_data_size + index_data_size, normals_data.data(), normals_data_size);

        // compress szSource into pchCompressed
        auto rawDataSize = (int)data.size();
        auto compressBufSize = LZ4_compressBound(rawDataSize);
        char* pchCompressed = new char[compressBufSize];
        memset(pchCompressed, compressBufSize, 0);
        int nCompressedSize = LZ4_compress_default((const char*)data.data(), pchCompressed, rawDataSize, compressBufSize);

        ofstream myfile;
        myfile.open(output);
        myfile << "// License: Apache 2.0. See LICENSE file in root directory.\n";
        myfile << "// Copyright(c) 2018 Intel Corporation. All Rights Reserved.\n\n";
        myfile << "// This file is auto-generated from " << name << ".obj\n";
        myfile << "#pragma once\n";
        myfile << "static uint32_t " << name << "_obj_data [] { ";

        auto nAllignedCompressedSize = nCompressedSize;
        auto leftover = nCompressedSize % 4;
        if (leftover % 4 != 0) nAllignedCompressedSize += (4 - leftover);

        for (int i = 0; i < nAllignedCompressedSize; i += 4)
        {
            uint32_t* ptr = (uint32_t*)(pchCompressed + i);
            myfile << "0x" << std::hex << (int)(*ptr);
            if (i < nAllignedCompressedSize - 1) myfile << ",";
        }

        myfile << "};\n";

        myfile << "#include <lz4.h>\n";
        myfile << "#include <vector>\n";

        myfile << "inline void uncompress_" << name << "_obj(std::vector<float3>& vertex_data, std::vector<float3>& normals, std::vector<short3>& index_data)\n";
        myfile << "{\n";
        myfile << "    std::vector<char> uncompressed(0x" << std::hex << rawDataSize << ", 0);\n";
        myfile << "    LZ4_decompress_safe((const char*)" << name << "_obj_data, uncompressed.data(), 0x" << std::hex << nCompressedSize << ", 0x" << std::hex << rawDataSize << ");\n";
        myfile << "    const int vertex_size = 0x" << std::hex << vertex_data.size() << " * sizeof(float3);\n";
        myfile << "    const int index_size = 0x" << std::hex << index_data.size() << " * sizeof(short3);\n";
        myfile << "    vertex_data.resize(0x" << std::hex << vertex_data.size() << ");\n";
        myfile << "    memcpy(vertex_data.data(), uncompressed.data(), vertex_size);\n";
        myfile << "    index_data.resize(0x" << std::hex << index_data.size() << ");\n";
        myfile << "    memcpy(index_data.data(), uncompressed.data() + vertex_size, index_size);\n";
        myfile << "    //normals.resize(0x" << std::hex << vertex_data.size() << ");\n";
        myfile << "    //memcpy(normals.data(), uncompressed.data() + vertex_size + index_size, vertex_size);\n";
        myfile << "}\n";

        myfile.close();
    }

    if (ends_with(input, ".png"))
    {
        ifstream ifs(input, ios::binary | ios::ate);
        ifstream::pos_type pos = ifs.tellg();

        std::vector<char> buffer(pos);

        ifs.seekg(0, ios::beg);
        ifs.read(&buffer[0], pos);

        ofstream myfile;
        myfile.open(output);
        myfile << "// License: Apache 2.0. See LICENSE file in root directory.\n";
        myfile << "// Copyright(c) 2018 Intel Corporation. All Rights Reserved.\n\n";
        myfile << "// This file is auto-generated from " << name << ".png\n";

        myfile << "static uint32_t " << name << "_png_size = 0x" << std::hex << buffer.size() << ";\n";

        myfile << "static uint8_t " << name << "_png_data [] { ";
        for (int i = 0; i < buffer.size(); i++)
        {
            uint8_t byte = buffer[i];
            myfile << "0x" << std::hex << (int)byte;
            if (i < buffer.size() - 1) myfile << ",";
        }
        myfile << "};\n";

        myfile.close();
    }

    return EXIT_SUCCESS;
}
catch (const exception& e)
{
    cerr << e.what() << endl;
    return EXIT_FAILURE;
}
catch (...)
{
    cerr << "some error" << endl;
    return EXIT_FAILURE;
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2002-2005 The Regents of The University of Michigan
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * Authors: Steve Reinhardt
 *          Nathan Binkert
 */

#ifndef __CPU_BASE_HH__
#define __CPU_BASE_HH__

#include <vector>

#include "arch/isa_traits.hh"
#include "base/statistics.hh"
#include "config/full_system.hh"
#include "sim/eventq.hh"
#include "sim/insttracer.hh"
#include "mem/mem_object.hh"

#if FULL_SYSTEM
#include "arch/interrupts.hh"
#endif

class BranchPred;
class CheckerCPU;
class ThreadContext;
class System;
class Port;

namespace TheISA
{
    class Predecoder;
}

class CPUProgressEvent : public Event
{
  protected:
    Tick interval;
    Counter lastNumInst;
    BaseCPU *cpu;

  public:
    CPUProgressEvent(EventQueue *q, Tick ival, BaseCPU *_cpu);

    void process();

    virtual const char *description();
};

class BaseCPU : public MemObject
{
  protected:
    // CPU's clock period in terms of the number of ticks of curTime.
    Tick clock;
    // @todo remove me after debugging with legion done
    Tick instCnt;

  public:
//    Tick currentTick;
    inline Tick frequency() const { return Clock::Frequency / clock; }
    inline Tick ticks(int numCycles) const { return clock * numCycles; }
    inline Tick curCycle() const { return curTick / clock; }
    inline Tick tickToCycles(Tick val) const { return val / clock; }
    // @todo remove me after debugging with legion done
    Tick instCount() { return instCnt; }

    /** The next cycle the CPU should be scheduled, given a cache
     * access or quiesce event returning on this cycle.  This function
     * may return curTick if the CPU should run on the current cycle.
     */
    Tick nextCycle();

    /** The next cycle the CPU should be scheduled, given a cache
     * access or quiesce event returning on the given Tick.  This
     * function may return curTick if the CPU should run on the
     * current cycle.
     * @param begin_tick The tick that the event is completing on.
     */
    Tick nextCycle(Tick begin_tick);

#if FULL_SYSTEM
  protected:
//    uint64_t interrupts[TheISA::NumInterruptLevels];
//    uint64_t intstatus;
    TheISA::Interrupts interrupts;

  public:
    virtual void post_interrupt(int int_num, int index);
    virtual void clear_interrupt(int int_num, int index);
    virtual void clear_interrupts();
    virtual uint64_t get_interrupts(int int_num);

    bool check_interrupts(ThreadContext * tc) const
    { return interrupts.check_interrupts(tc); }

    class ProfileEvent : public Event
    {
      private:
        BaseCPU *cpu;
        int interval;

      public:
        ProfileEvent(BaseCPU *cpu, int interval);
        void process();
    };
    ProfileEvent *profileEvent;
#endif

  protected:
    std::vector<ThreadContext *> threadContexts;
    std::vector<TheISA::Predecoder *> predecoders;

    Trace::InstTracer * tracer;

  public:

    /// Provide access to the tracer pointer
    Trace::InstTracer * getTracer() { return tracer; }

    /// Notify the CPU that the indicated context is now active.  The
    /// delay parameter indicates the number of ticks to wait before
    /// executing (typically 0 or 1).
    virtual void activateContext(int thread_num, int delay) {}

    /// Notify the CPU that the indicated context is now suspended.
    virtual void suspendContext(int thread_num) {}

    /// Notify the CPU that the indicated context is now deallocated.
    virtual void deallocateContext(int thread_num) {}

    /// Notify the CPU that the indicated context is now halted.
    virtual void haltContext(int thread_num) {}

   /// Given a Thread Context pointer return the thread num
   int findContext(ThreadContext *tc);

   /// Given a thread num get tho thread context for it
   ThreadContext *getContext(int tn) { return threadContexts[tn]; }

  public:
    struct Params
    {
        std::string name;
        int numberOfThreads;
        bool deferRegistration;
        Counter max_insts_any_thread;
        Counter max_insts_all_threads;
        Counter max_loads_any_thread;
        Counter max_loads_all_threads;
        Tick clock;
        bool functionTrace;
        Tick functionTraceStart;
        System *system;
        int cpu_id;
        Trace::InstTracer * tracer;

        Tick phase;
#if FULL_SYSTEM
        Tick profile;

        bool do_statistics_insts;
        bool do_checkpoint_insts;
        bool do_quiesce;
#endif
        Tick progress_interval;
        BaseCPU *checker;

#if THE_ISA == MIPS_ISA
      /* Note: It looks like it will be better to allow simulator users
         to specify the values of individual variables instead of requiring
         users to define the values of entire registers
         Especially since a lot of these variables can be created from other
         user parameters  (cache descriptions)
                                               -jpp
      */
      // MIPS CP0 State - First individual variables
      // Page numbers refer to revision 2.50 (July 2005) of the MIPS32 ARM, Volume III (PRA)
      unsigned CP0_IntCtl_IPTI; // Page 93, IP Timer Interrupt
      unsigned CP0_IntCtl_IPPCI; // Page 94, IP Performance Counter Interrupt
      unsigned CP0_SrsCtl_HSS; // Page 95, Highest Implemented Shadow Set
      unsigned CP0_PRId_CompanyOptions; // Page 105, Manufacture options
      unsigned CP0_PRId_CompanyID; // Page 105, Company ID - (0-255, 1=>MIPS)
      unsigned CP0_PRId_ProcessorID; // Page 105
      unsigned CP0_PRId_Revision; // Page 105
      unsigned CP0_EBase_CPUNum; // Page 106, CPU Number in a multiprocessor system
      unsigned CP0_Config_BE; // Page 108, Big/Little Endian mode
      unsigned CP0_Config_AT; //Page 109
      unsigned CP0_Config_AR; //Page 109
      unsigned CP0_Config_MT; //Page 109
      unsigned CP0_Config_VI; //Page 109
      unsigned CP0_Config1_M; // Page 110
      unsigned CP0_Config1_MMU; // Page 110
      unsigned CP0_Config1_IS; // Page 110
      unsigned CP0_Config1_IL; // Page 111
      unsigned CP0_Config1_IA; // Page 111
      unsigned CP0_Config1_DS; // Page 111
      unsigned CP0_Config1_DL; // Page 112
      unsigned CP0_Config1_DA; // Page 112
      bool CP0_Config1_C2; // Page 112
      bool CP0_Config1_MD;// Page 112 - Technically not used in MIPS32
      bool CP0_Config1_PC;// Page 112
      bool CP0_Config1_WR;// Page 113
      bool CP0_Config1_CA;// Page 113
      bool CP0_Config1_EP;// Page 113
      bool CP0_Config1_FP;// Page 113
      bool CP0_Config2_M; // Page 114
      unsigned CP0_Config2_TU;// Page 114
      unsigned CP0_Config2_TS;// Page 114
      unsigned CP0_Config2_TL;// Page 115
      unsigned CP0_Config2_TA;// Page 115
      unsigned CP0_Config2_SU;// Page 115
      unsigned CP0_Config2_SS;// Page 115
      unsigned CP0_Config2_SL;// Page 116
      unsigned CP0_Config2_SA;// Page 116
      bool CP0_Config3_M; //// Page 117
      bool CP0_Config3_DSPP;// Page 117
      bool CP0_Config3_LPA;// Page 117
      bool CP0_Config3_VEIC;// Page 118
      bool CP0_Config3_VInt; // Page 118
      bool CP0_Config3_SP;// Page 118
      bool CP0_Config3_MT;// Page 119
      bool CP0_Config3_SM;// Page 119
      bool CP0_Config3_TL;// Page 119

      bool CP0_WatchHi_M; // Page 124
      bool CP0_PerfCtr_M; // Page 130
      bool CP0_PerfCtr_W; // Page 130

      // Then, whole registers
      unsigned CP0_PRId;
      unsigned CP0_Config;
      unsigned CP0_Config1;
      unsigned CP0_Config2;
      unsigned CP0_Config3;

#endif

        Params();
    };

    const Params *params;

    BaseCPU(Params *params);
    virtual ~BaseCPU();

    virtual void init();
    virtual void startup();
    virtual void regStats();

    virtual void activateWhenReady(int tid) {};

    void registerThreadContexts();

    /// Prepare for another CPU to take over execution.  When it is
    /// is ready (drained pipe) it signals the sampler.
    virtual void switchOut();

    /// Take over execution from the given CPU.  Used for warm-up and
    /// sampling.
    virtual void takeOverFrom(BaseCPU *, Port *ic, Port *dc);

    /**
     *  Number of threads we're actually simulating (<= SMT_MAX_THREADS).
     * This is a constant for the duration of the simulation.
     */
    int number_of_threads;

    /**
     * Vector of per-thread instruction-based event queues.  Used for
     * scheduling events based on number of instructions committed by
     * a particular thread.
     */
    EventQueue **comInstEventQueue;

    /**
     * Vector of per-thread load-based event queues.  Used for
     * scheduling events based on number of loads committed by
     *a particular thread.
     */
    EventQueue **comLoadEventQueue;

    System *system;

    Tick phase;

#if FULL_SYSTEM
    /**
     * Serialize this object to the given output stream.
     * @param os The stream to serialize to.
     */
    virtual void serialize(std::ostream &os);

    /**
     * Reconstruct the state of this object from a checkpoint.
     * @param cp The checkpoint use.
     * @param section The section name of this object
     */
    virtual void unserialize(Checkpoint *cp, const std::string &section);

#endif

    /**
     * Return pointer to CPU's branch predictor (NULL if none).
     * @return Branch predictor pointer.
     */
    virtual BranchPred *getBranchPred() { return NULL; };

    virtual Counter totalInstructions() const { return 0; }

    // Function tracing
  private:
    bool functionTracingEnabled;
    std::ostream *functionTraceStream;
    Addr currentFunctionStart;
    Addr currentFunctionEnd;
    Tick functionEntryTick;
    void enableFunctionTrace();
    void traceFunctionsInternal(Addr pc);

  protected:
    void traceFunctions(Addr pc)
    {
        if (functionTracingEnabled)
            traceFunctionsInternal(pc);
    }

  private:
    static std::vector<BaseCPU *> cpuList;   //!< Static global cpu list

  public:
    static int numSimulatedCPUs() { return cpuList.size(); }
    static Counter numSimulatedInstructions()
    {
        Counter total = 0;

        int size = cpuList.size();
        for (int i = 0; i < size; ++i)
            total += cpuList[i]->totalInstructions();

        return total;
    }

  public:
    // Number of CPU cycles simulated
    Stats::Scalar<> numCycles;
};

#endif // __CPU_BASE_HH__
<commit_msg>add core specific parameter to BaseCPU params<commit_after>/*
 * Copyright (c) 2002-2005 The Regents of The University of Michigan
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * Authors: Steve Reinhardt
 *          Nathan Binkert
 */

#ifndef __CPU_BASE_HH__
#define __CPU_BASE_HH__

#include <vector>

#include "arch/isa_traits.hh"
#include "base/statistics.hh"
#include "config/full_system.hh"
#include "sim/eventq.hh"
#include "sim/insttracer.hh"
#include "mem/mem_object.hh"

#if FULL_SYSTEM
#include "arch/interrupts.hh"
#endif

class BranchPred;
class CheckerCPU;
class ThreadContext;
class System;
class Port;

namespace TheISA
{
    class Predecoder;
}

class CPUProgressEvent : public Event
{
  protected:
    Tick interval;
    Counter lastNumInst;
    BaseCPU *cpu;

  public:
    CPUProgressEvent(EventQueue *q, Tick ival, BaseCPU *_cpu);

    void process();

    virtual const char *description();
};

class BaseCPU : public MemObject
{
  protected:
    // CPU's clock period in terms of the number of ticks of curTime.
    Tick clock;
    // @todo remove me after debugging with legion done
    Tick instCnt;

  public:
//    Tick currentTick;
    inline Tick frequency() const { return Clock::Frequency / clock; }
    inline Tick ticks(int numCycles) const { return clock * numCycles; }
    inline Tick curCycle() const { return curTick / clock; }
    inline Tick tickToCycles(Tick val) const { return val / clock; }
    // @todo remove me after debugging with legion done
    Tick instCount() { return instCnt; }

    /** The next cycle the CPU should be scheduled, given a cache
     * access or quiesce event returning on this cycle.  This function
     * may return curTick if the CPU should run on the current cycle.
     */
    Tick nextCycle();

    /** The next cycle the CPU should be scheduled, given a cache
     * access or quiesce event returning on the given Tick.  This
     * function may return curTick if the CPU should run on the
     * current cycle.
     * @param begin_tick The tick that the event is completing on.
     */
    Tick nextCycle(Tick begin_tick);

#if FULL_SYSTEM
  protected:
//    uint64_t interrupts[TheISA::NumInterruptLevels];
//    uint64_t intstatus;
    TheISA::Interrupts interrupts;

  public:
    virtual void post_interrupt(int int_num, int index);
    virtual void clear_interrupt(int int_num, int index);
    virtual void clear_interrupts();
    virtual uint64_t get_interrupts(int int_num);

    bool check_interrupts(ThreadContext * tc) const
    { return interrupts.check_interrupts(tc); }

    class ProfileEvent : public Event
    {
      private:
        BaseCPU *cpu;
        int interval;

      public:
        ProfileEvent(BaseCPU *cpu, int interval);
        void process();
    };
    ProfileEvent *profileEvent;
#endif

  protected:
    std::vector<ThreadContext *> threadContexts;
    std::vector<TheISA::Predecoder *> predecoders;

    Trace::InstTracer * tracer;

  public:

    /// Provide access to the tracer pointer
    Trace::InstTracer * getTracer() { return tracer; }

    /// Notify the CPU that the indicated context is now active.  The
    /// delay parameter indicates the number of ticks to wait before
    /// executing (typically 0 or 1).
    virtual void activateContext(int thread_num, int delay) {}

    /// Notify the CPU that the indicated context is now suspended.
    virtual void suspendContext(int thread_num) {}

    /// Notify the CPU that the indicated context is now deallocated.
    virtual void deallocateContext(int thread_num) {}

    /// Notify the CPU that the indicated context is now halted.
    virtual void haltContext(int thread_num) {}

   /// Given a Thread Context pointer return the thread num
   int findContext(ThreadContext *tc);

   /// Given a thread num get tho thread context for it
   ThreadContext *getContext(int tn) { return threadContexts[tn]; }

  public:
    struct Params
    {
        std::string name;
        int numberOfThreads;
        bool deferRegistration;
        Counter max_insts_any_thread;
        Counter max_insts_all_threads;
        Counter max_loads_any_thread;
        Counter max_loads_all_threads;
        Tick clock;
        bool functionTrace;
        Tick functionTraceStart;
        System *system;
        int cpu_id;
        Trace::InstTracer * tracer;

        Tick phase;
#if FULL_SYSTEM
        Tick profile;

        bool do_statistics_insts;
        bool do_checkpoint_insts;
        bool do_quiesce;
#endif
        Tick progress_interval;
        BaseCPU *checker;

        TheISA::CoreSpecific coreParams; //ISA-Specific Params That Set Up State in Core

        Params();
    };

    const Params *params;

    BaseCPU(Params *params);
    virtual ~BaseCPU();

    virtual void init();
    virtual void startup();
    virtual void regStats();

    virtual void activateWhenReady(int tid) {};

    void registerThreadContexts();

    /// Prepare for another CPU to take over execution.  When it is
    /// is ready (drained pipe) it signals the sampler.
    virtual void switchOut();

    /// Take over execution from the given CPU.  Used for warm-up and
    /// sampling.
    virtual void takeOverFrom(BaseCPU *, Port *ic, Port *dc);

    /**
     *  Number of threads we're actually simulating (<= SMT_MAX_THREADS).
     * This is a constant for the duration of the simulation.
     */
    int number_of_threads;

    /**
     * Vector of per-thread instruction-based event queues.  Used for
     * scheduling events based on number of instructions committed by
     * a particular thread.
     */
    EventQueue **comInstEventQueue;

    /**
     * Vector of per-thread load-based event queues.  Used for
     * scheduling events based on number of loads committed by
     *a particular thread.
     */
    EventQueue **comLoadEventQueue;

    System *system;

    Tick phase;

#if FULL_SYSTEM
    /**
     * Serialize this object to the given output stream.
     * @param os The stream to serialize to.
     */
    virtual void serialize(std::ostream &os);

    /**
     * Reconstruct the state of this object from a checkpoint.
     * @param cp The checkpoint use.
     * @param section The section name of this object
     */
    virtual void unserialize(Checkpoint *cp, const std::string &section);

#endif

    /**
     * Return pointer to CPU's branch predictor (NULL if none).
     * @return Branch predictor pointer.
     */
    virtual BranchPred *getBranchPred() { return NULL; };

    virtual Counter totalInstructions() const { return 0; }

    // Function tracing
  private:
    bool functionTracingEnabled;
    std::ostream *functionTraceStream;
    Addr currentFunctionStart;
    Addr currentFunctionEnd;
    Tick functionEntryTick;
    void enableFunctionTrace();
    void traceFunctionsInternal(Addr pc);

  protected:
    void traceFunctions(Addr pc)
    {
        if (functionTracingEnabled)
            traceFunctionsInternal(pc);
    }

  private:
    static std::vector<BaseCPU *> cpuList;   //!< Static global cpu list

  public:
    static int numSimulatedCPUs() { return cpuList.size(); }
    static Counter numSimulatedInstructions()
    {
        Counter total = 0;

        int size = cpuList.size();
        for (int i = 0; i < size; ++i)
            total += cpuList[i]->totalInstructions();

        return total;
    }

  public:
    // Number of CPU cycles simulated
    Stats::Scalar<> numCycles;
};

#endif // __CPU_BASE_HH__
<|endoftext|>


<commit_before>// Copyright 2017 The Ray Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "ray/gcs/gcs_client/global_state_accessor.h"

#include "gtest/gtest.h"
#include "ray/common/asio/instrumented_io_context.h"
#include "ray/common/test_util.h"
#include "ray/gcs/gcs_server/gcs_server.h"
#include "ray/gcs/test/gcs_test_util.h"
#include "ray/rpc/gcs_server/gcs_rpc_client.h"

namespace ray {

class GlobalStateAccessorTest : public ::testing::Test {
 public:
  GlobalStateAccessorTest() { TestSetupUtil::StartUpRedisServers(std::vector<int>()); }

  virtual ~GlobalStateAccessorTest() { TestSetupUtil::ShutDownRedisServers(); }

 protected:
  void SetUp() override {
    RayConfig::instance().gcs_max_active_rpcs_per_handler() = -1;
    config.grpc_server_port = 0;
    config.grpc_server_name = "MockedGcsServer";
    config.grpc_server_thread_num = 1;
    config.redis_address = "127.0.0.1";
    config.node_ip_address = "127.0.0.1";
    config.enable_sharding_conn = false;
    config.redis_port = TEST_REDIS_SERVER_PORTS.front();

    io_service_.reset(new instrumented_io_context());
    gcs_server_.reset(new gcs::GcsServer(config, *io_service_));
    gcs_server_->Start();
    work_ = std::make_unique<boost::asio::io_service::work>(*io_service_);
    thread_io_service_.reset(new std::thread([this] { io_service_->run(); }));

    // Wait until server starts listening.
    while (!gcs_server_->IsStarted()) {
      std::this_thread::sleep_for(std::chrono::milliseconds(10));
    }

    // Create GCS client.
    gcs::GcsClientOptions options(config.redis_address, config.redis_port,
                                  config.redis_password);
    gcs_client_ = std::make_unique<gcs::GcsClient>(options);
    RAY_CHECK_OK(gcs_client_->Connect(*io_service_));

    // Create global state.
    global_state_ = std::make_unique<gcs::GlobalStateAccessor>(options);
    RAY_CHECK(global_state_->Connect());
  }

  void TearDown() override {
    global_state_->Disconnect();
    global_state_.reset();

    gcs_client_->Disconnect();
    gcs_client_.reset();

    gcs_server_->Stop();
    TestSetupUtil::FlushAllRedisServers();

    io_service_->stop();
    thread_io_service_->join();
    gcs_server_.reset();
  }

  // GCS server.
  gcs::GcsServerConfig config;
  std::unique_ptr<gcs::GcsServer> gcs_server_;
  std::unique_ptr<std::thread> thread_io_service_;
  std::unique_ptr<instrumented_io_context> io_service_;

  // GCS client.
  std::unique_ptr<gcs::GcsClient> gcs_client_;

  std::unique_ptr<gcs::GlobalStateAccessor> global_state_;

  // Timeout waiting for GCS server reply, default is 2s.
  const std::chrono::milliseconds timeout_ms_{2000};
  std::unique_ptr<boost::asio::io_service::work> work_;
};

TEST_F(GlobalStateAccessorTest, TestJobTable) {
  int job_count = 100;
  ASSERT_EQ(global_state_->GetAllJobInfo().size(), 0);
  for (int index = 0; index < job_count; ++index) {
    auto job_id = JobID::FromInt(index);
    auto job_table_data = Mocker::GenJobTableData(job_id);
    std::promise<bool> promise;
    RAY_CHECK_OK(gcs_client_->Jobs().AsyncAdd(
        job_table_data, [&promise](Status status) { promise.set_value(status.ok()); }));
    promise.get_future().get();
  }
  ASSERT_EQ(global_state_->GetAllJobInfo().size(), job_count);
}

TEST_F(GlobalStateAccessorTest, TestNodeTable) {
  int node_count = 100;
  ASSERT_EQ(global_state_->GetAllNodeInfo().size(), 0);
  // It's useful to check if index value will be marked as address suffix.
  for (int index = 0; index < node_count; ++index) {
    auto node_table_data =
        Mocker::GenNodeInfo(index, std::string("127.0.0.") + std::to_string(index));
    std::promise<bool> promise;
    RAY_CHECK_OK(gcs_client_->Nodes().AsyncRegister(
        *node_table_data, [&promise](Status status) { promise.set_value(status.ok()); }));
    WaitReady(promise.get_future(), timeout_ms_);
  }
  auto node_table = global_state_->GetAllNodeInfo();
  ASSERT_EQ(node_table.size(), node_count);
  for (int index = 0; index < node_count; ++index) {
    rpc::GcsNodeInfo node_data;
    node_data.ParseFromString(node_table[index]);
    ASSERT_EQ(node_data.node_manager_address(),
              std::string("127.0.0.") + std::to_string(node_data.node_manager_port()));
  }
}

TEST_F(GlobalStateAccessorTest, TestNodeResourceTable) {
  int node_count = 100;
  ASSERT_EQ(global_state_->GetAllNodeInfo().size(), 0);
  for (int index = 0; index < node_count; ++index) {
    auto node_table_data =
        Mocker::GenNodeInfo(index, std::string("127.0.0.") + std::to_string(index));
    auto node_id = NodeID::FromBinary(node_table_data->node_id());
    std::promise<bool> promise;
    RAY_CHECK_OK(gcs_client_->Nodes().AsyncRegister(
        *node_table_data, [&promise](Status status) { promise.set_value(status.ok()); }));
    WaitReady(promise.get_future(), timeout_ms_);
    ray::gcs::NodeResourceInfoAccessor::ResourceMap resources;
    rpc::ResourceTableData resource_table_data;
    resource_table_data.set_resource_capacity(static_cast<double>(index + 1) + 0.1);
    resources[std::to_string(index)] =
        std::make_shared<rpc::ResourceTableData>(resource_table_data);
    RAY_IGNORE_EXPR(gcs_client_->NodeResources().AsyncUpdateResources(
        node_id, resources, [](Status status) { RAY_CHECK(status.ok()); }));
  }
  auto node_table = global_state_->GetAllNodeInfo();
  ASSERT_EQ(node_table.size(), node_count);
  for (int index = 0; index < node_count; ++index) {
    rpc::GcsNodeInfo node_data;
    node_data.ParseFromString(node_table[index]);
    auto resource_map_str =
        global_state_->GetNodeResourceInfo(NodeID::FromBinary(node_data.node_id()));
    rpc::ResourceMap resource_map;
    resource_map.ParseFromString(resource_map_str);
    ASSERT_EQ(
        static_cast<uint32_t>(
            (*resource_map.mutable_items())[std::to_string(node_data.node_manager_port())]
                .resource_capacity()),
        node_data.node_manager_port() + 1);
  }
}

TEST_F(GlobalStateAccessorTest, TestGetAllResourceUsage) {
  std::unique_ptr<std::string> resources = global_state_->GetAllResourceUsage();
  rpc::ResourceUsageBatchData resource_usage_batch_data;
  resource_usage_batch_data.ParseFromString(*resources.get());
  ASSERT_EQ(resource_usage_batch_data.batch_size(), 0);

  auto node_table_data = Mocker::GenNodeInfo();
  std::promise<bool> promise;
  RAY_CHECK_OK(gcs_client_->Nodes().AsyncRegister(
      *node_table_data, [&promise](Status status) { promise.set_value(status.ok()); }));
  WaitReady(promise.get_future(), timeout_ms_);
  auto node_table = global_state_->GetAllNodeInfo();
  ASSERT_EQ(node_table.size(), 1);

  // Report resource usage first time.
  std::promise<bool> promise1;
  auto resources1 = std::make_shared<rpc::ResourcesData>();
  resources1->set_node_id(node_table_data->node_id());
  RAY_CHECK_OK(gcs_client_->NodeResources().AsyncReportResourceUsage(
      resources1, [&promise1](Status status) { promise1.set_value(status.ok()); }));
  WaitReady(promise1.get_future(), timeout_ms_);

  resources = global_state_->GetAllResourceUsage();
  resource_usage_batch_data.ParseFromString(*resources.get());
  ASSERT_EQ(resource_usage_batch_data.batch_size(), 1);

  // Report changed resource usage.
  std::promise<bool> promise2;
  auto heartbeat2 = std::make_shared<rpc::ResourcesData>();
  heartbeat2->set_node_id(node_table_data->node_id());
  (*heartbeat2->mutable_resources_total())["CPU"] = 1;
  (*heartbeat2->mutable_resources_total())["GPU"] = 10;
  heartbeat2->set_resources_available_changed(true);
  (*heartbeat2->mutable_resources_available())["GPU"] = 5;
  RAY_CHECK_OK(gcs_client_->NodeResources().AsyncReportResourceUsage(
      heartbeat2, [&promise2](Status status) { promise2.set_value(status.ok()); }));
  WaitReady(promise2.get_future(), timeout_ms_);

  resources = global_state_->GetAllResourceUsage();
  resource_usage_batch_data.ParseFromString(*resources.get());
  ASSERT_EQ(resource_usage_batch_data.batch_size(), 1);
  auto resources_data = resource_usage_batch_data.mutable_batch()->at(0);
  ASSERT_EQ(resources_data.resources_total_size(), 2);
  ASSERT_EQ((*resources_data.mutable_resources_total())["CPU"], 1.0);
  ASSERT_EQ((*resources_data.mutable_resources_total())["GPU"], 10.0);
  ASSERT_EQ(resources_data.resources_available_size(), 1);
  ASSERT_EQ((*resources_data.mutable_resources_available())["GPU"], 5.0);

  // Report unchanged resource usage. (Only works with light resource usage report
  // enabled)
  std::promise<bool> promise3;
  auto heartbeat3 = std::make_shared<rpc::ResourcesData>();
  heartbeat3->set_node_id(node_table_data->node_id());
  (*heartbeat3->mutable_resources_available())["CPU"] = 1;
  (*heartbeat3->mutable_resources_available())["GPU"] = 6;
  RAY_CHECK_OK(gcs_client_->NodeResources().AsyncReportResourceUsage(
      heartbeat3, [&promise3](Status status) { promise3.set_value(status.ok()); }));
  WaitReady(promise3.get_future(), timeout_ms_);

  resources = global_state_->GetAllResourceUsage();
  resource_usage_batch_data.ParseFromString(*resources.get());
  ASSERT_EQ(resource_usage_batch_data.batch_size(), 1);
  resources_data = resource_usage_batch_data.mutable_batch()->at(0);
  ASSERT_EQ(resources_data.resources_total_size(), 2);
  ASSERT_EQ((*resources_data.mutable_resources_total())["CPU"], 1.0);
  ASSERT_EQ((*resources_data.mutable_resources_total())["GPU"], 10.0);
  ASSERT_EQ(resources_data.resources_available_size(), 1);
  ASSERT_EQ((*resources_data.mutable_resources_available())["GPU"], 5.0);
}

TEST_F(GlobalStateAccessorTest, TestProfileTable) {
  int profile_count = RayConfig::instance().maximum_profile_table_rows_count() + 1;
  ASSERT_EQ(global_state_->GetAllProfileInfo().size(), 0);
  for (int index = 0; index < profile_count; ++index) {
    auto node_id = NodeID::FromRandom();
    auto profile_table_data = Mocker::GenProfileTableData(node_id);
    std::promise<bool> promise;
    RAY_CHECK_OK(gcs_client_->Stats().AsyncAddProfileData(
        profile_table_data,
        [&promise](Status status) { promise.set_value(status.ok()); }));
    WaitReady(promise.get_future(), timeout_ms_);
  }
  ASSERT_EQ(global_state_->GetAllProfileInfo().size(),
            RayConfig::instance().maximum_profile_table_rows_count());
}

TEST_F(GlobalStateAccessorTest, TestWorkerTable) {
  ASSERT_EQ(global_state_->GetAllWorkerInfo().size(), 0);
  // Add worker info
  auto worker_table_data = Mocker::GenWorkerTableData();
  worker_table_data->mutable_worker_address()->set_worker_id(
      WorkerID::FromRandom().Binary());
  ASSERT_TRUE(global_state_->AddWorkerInfo(worker_table_data->SerializeAsString()));

  // Get worker info
  auto worker_id = WorkerID::FromBinary(worker_table_data->worker_address().worker_id());
  ASSERT_TRUE(global_state_->GetWorkerInfo(worker_id));

  // Add another worker info
  auto another_worker_data = Mocker::GenWorkerTableData();
  another_worker_data->mutable_worker_address()->set_worker_id(
      WorkerID::FromRandom().Binary());
  ASSERT_TRUE(global_state_->AddWorkerInfo(another_worker_data->SerializeAsString()));
  ASSERT_EQ(global_state_->GetAllWorkerInfo().size(), 2);
}

// TODO(sang): Add tests after adding asyncAdd
TEST_F(GlobalStateAccessorTest, TestPlacementGroupTable) {
  ASSERT_EQ(global_state_->GetAllPlacementGroupInfo().size(), 0);
}

}  // namespace ray

int main(int argc, char **argv) {
  ray::RayLog::InstallFailureSignalHandler(argv[0]);
  InitShutdownRAII ray_log_shutdown_raii(ray::RayLog::StartRayLog,
                                         ray::RayLog::ShutDownRayLog, argv[0],
                                         ray::RayLogLevel::INFO,
                                         /*log_dir=*/"");
  ::testing::InitGoogleTest(&argc, argv);
  RAY_CHECK(argc == 3);
  ray::TEST_REDIS_SERVER_EXEC_PATH = argv[1];
  ray::TEST_REDIS_CLIENT_EXEC_PATH = argv[2];
  return RUN_ALL_TESTS();
}
<commit_msg>[gcs/ha] Fix global_state_accessor_test to support redisless mode (#21729)<commit_after>// Copyright 2017 The Ray Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "ray/gcs/gcs_client/global_state_accessor.h"

#include "gtest/gtest.h"
#include "ray/common/asio/instrumented_io_context.h"
#include "ray/common/test_util.h"
#include "ray/gcs/gcs_server/gcs_server.h"
#include "ray/gcs/test/gcs_test_util.h"
#include "ray/rpc/gcs_server/gcs_rpc_client.h"

namespace ray {

class GlobalStateAccessorTest : public ::testing::TestWithParam<bool> {
 public:
  GlobalStateAccessorTest() {
    if (GetParam()) {
      RayConfig::instance().bootstrap_with_gcs() = true;
      RayConfig::instance().gcs_storage() = "memory";
      RayConfig::instance().gcs_grpc_based_pubsub() = true;
    } else {
      RayConfig::instance().bootstrap_with_gcs() = false;
      RayConfig::instance().gcs_storage() = "redis";
      RayConfig::instance().gcs_grpc_based_pubsub() = false;
    }

    if (!RayConfig::instance().bootstrap_with_gcs()) {
      TestSetupUtil::StartUpRedisServers(std::vector<int>());
    }
  }

  virtual ~GlobalStateAccessorTest() {
    if (!RayConfig::instance().bootstrap_with_gcs()) {
      TestSetupUtil::ShutDownRedisServers();
    }
  }

 protected:
  void SetUp() override {
    RayConfig::instance().gcs_max_active_rpcs_per_handler() = -1;
    if (RayConfig::instance().bootstrap_with_gcs()) {
      config.grpc_server_port = 6379;
      config.grpc_pubsub_enabled = true;
    } else {
      config.grpc_server_port = 0;
      config.redis_address = "127.0.0.1";
      config.enable_sharding_conn = false;
      config.grpc_pubsub_enabled = false;
      config.redis_port = TEST_REDIS_SERVER_PORTS.front();
    }

    config.node_ip_address = "127.0.0.1";
    config.grpc_server_name = "MockedGcsServer";
    config.grpc_server_thread_num = 1;

    io_service_.reset(new instrumented_io_context());
    gcs_server_.reset(new gcs::GcsServer(config, *io_service_));
    gcs_server_->Start();
    work_ = std::make_unique<boost::asio::io_service::work>(*io_service_);
    thread_io_service_.reset(new std::thread([this] { io_service_->run(); }));

    // Wait until server starts listening.
    while (!gcs_server_->IsStarted()) {
      std::this_thread::sleep_for(std::chrono::milliseconds(10));
    }

    // Create GCS client and global state.
    if (RayConfig::instance().bootstrap_with_gcs()) {
      gcs::GcsClientOptions options("127.0.0.1:6379");
      gcs_client_ = std::make_unique<gcs::GcsClient>(options);
      global_state_ = std::make_unique<gcs::GlobalStateAccessor>(options);
    } else {
      gcs::GcsClientOptions options(config.redis_address, config.redis_port,
                                    config.redis_password);
      gcs_client_ = std::make_unique<gcs::GcsClient>(options);
      global_state_ = std::make_unique<gcs::GlobalStateAccessor>(options);
    }
    RAY_CHECK_OK(gcs_client_->Connect(*io_service_));

    RAY_CHECK(global_state_->Connect());
  }

  void TearDown() override {
    global_state_->Disconnect();
    global_state_.reset();

    gcs_client_->Disconnect();
    gcs_client_.reset();

    gcs_server_->Stop();
    if (!RayConfig::instance().bootstrap_with_gcs()) {
      TestSetupUtil::FlushAllRedisServers();
    }

    io_service_->stop();
    thread_io_service_->join();
    gcs_server_.reset();
  }

  // GCS server.
  gcs::GcsServerConfig config;
  std::unique_ptr<gcs::GcsServer> gcs_server_;
  std::unique_ptr<std::thread> thread_io_service_;
  std::unique_ptr<instrumented_io_context> io_service_;

  // GCS client.
  std::unique_ptr<gcs::GcsClient> gcs_client_;

  std::unique_ptr<gcs::GlobalStateAccessor> global_state_;

  // Timeout waiting for GCS server reply, default is 2s.
  const std::chrono::milliseconds timeout_ms_{2000};
  std::unique_ptr<boost::asio::io_service::work> work_;
};

TEST_P(GlobalStateAccessorTest, TestJobTable) {
  int job_count = 100;
  ASSERT_EQ(global_state_->GetAllJobInfo().size(), 0);
  for (int index = 0; index < job_count; ++index) {
    auto job_id = JobID::FromInt(index);
    auto job_table_data = Mocker::GenJobTableData(job_id);
    std::promise<bool> promise;
    RAY_CHECK_OK(gcs_client_->Jobs().AsyncAdd(
        job_table_data, [&promise](Status status) { promise.set_value(status.ok()); }));
    promise.get_future().get();
  }
  ASSERT_EQ(global_state_->GetAllJobInfo().size(), job_count);
}

TEST_P(GlobalStateAccessorTest, TestNodeTable) {
  int node_count = 100;
  ASSERT_EQ(global_state_->GetAllNodeInfo().size(), 0);
  // It's useful to check if index value will be marked as address suffix.
  for (int index = 0; index < node_count; ++index) {
    auto node_table_data =
        Mocker::GenNodeInfo(index, std::string("127.0.0.") + std::to_string(index));
    std::promise<bool> promise;
    RAY_CHECK_OK(gcs_client_->Nodes().AsyncRegister(
        *node_table_data, [&promise](Status status) { promise.set_value(status.ok()); }));
    WaitReady(promise.get_future(), timeout_ms_);
  }
  auto node_table = global_state_->GetAllNodeInfo();
  ASSERT_EQ(node_table.size(), node_count);
  for (int index = 0; index < node_count; ++index) {
    rpc::GcsNodeInfo node_data;
    node_data.ParseFromString(node_table[index]);
    ASSERT_EQ(node_data.node_manager_address(),
              std::string("127.0.0.") + std::to_string(node_data.node_manager_port()));
  }
}

TEST_P(GlobalStateAccessorTest, TestNodeResourceTable) {
  int node_count = 100;
  ASSERT_EQ(global_state_->GetAllNodeInfo().size(), 0);
  for (int index = 0; index < node_count; ++index) {
    auto node_table_data =
        Mocker::GenNodeInfo(index, std::string("127.0.0.") + std::to_string(index));
    auto node_id = NodeID::FromBinary(node_table_data->node_id());
    std::promise<bool> promise;
    RAY_CHECK_OK(gcs_client_->Nodes().AsyncRegister(
        *node_table_data, [&promise](Status status) { promise.set_value(status.ok()); }));
    WaitReady(promise.get_future(), timeout_ms_);
    ray::gcs::NodeResourceInfoAccessor::ResourceMap resources;
    rpc::ResourceTableData resource_table_data;
    resource_table_data.set_resource_capacity(static_cast<double>(index + 1) + 0.1);
    resources[std::to_string(index)] =
        std::make_shared<rpc::ResourceTableData>(resource_table_data);
    RAY_IGNORE_EXPR(gcs_client_->NodeResources().AsyncUpdateResources(
        node_id, resources, [](Status status) { RAY_CHECK(status.ok()); }));
  }
  auto node_table = global_state_->GetAllNodeInfo();
  ASSERT_EQ(node_table.size(), node_count);
  for (int index = 0; index < node_count; ++index) {
    rpc::GcsNodeInfo node_data;
    node_data.ParseFromString(node_table[index]);
    auto resource_map_str =
        global_state_->GetNodeResourceInfo(NodeID::FromBinary(node_data.node_id()));
    rpc::ResourceMap resource_map;
    resource_map.ParseFromString(resource_map_str);
    ASSERT_EQ(
        static_cast<uint32_t>(
            (*resource_map.mutable_items())[std::to_string(node_data.node_manager_port())]
                .resource_capacity()),
        node_data.node_manager_port() + 1);
  }
}

TEST_P(GlobalStateAccessorTest, TestGetAllResourceUsage) {
  std::unique_ptr<std::string> resources = global_state_->GetAllResourceUsage();
  rpc::ResourceUsageBatchData resource_usage_batch_data;
  resource_usage_batch_data.ParseFromString(*resources.get());
  ASSERT_EQ(resource_usage_batch_data.batch_size(), 0);

  auto node_table_data = Mocker::GenNodeInfo();
  std::promise<bool> promise;
  RAY_CHECK_OK(gcs_client_->Nodes().AsyncRegister(
      *node_table_data, [&promise](Status status) { promise.set_value(status.ok()); }));
  WaitReady(promise.get_future(), timeout_ms_);
  auto node_table = global_state_->GetAllNodeInfo();
  ASSERT_EQ(node_table.size(), 1);

  // Report resource usage first time.
  std::promise<bool> promise1;
  auto resources1 = std::make_shared<rpc::ResourcesData>();
  resources1->set_node_id(node_table_data->node_id());
  RAY_CHECK_OK(gcs_client_->NodeResources().AsyncReportResourceUsage(
      resources1, [&promise1](Status status) { promise1.set_value(status.ok()); }));
  WaitReady(promise1.get_future(), timeout_ms_);

  resources = global_state_->GetAllResourceUsage();
  resource_usage_batch_data.ParseFromString(*resources.get());
  ASSERT_EQ(resource_usage_batch_data.batch_size(), 1);

  // Report changed resource usage.
  std::promise<bool> promise2;
  auto heartbeat2 = std::make_shared<rpc::ResourcesData>();
  heartbeat2->set_node_id(node_table_data->node_id());
  (*heartbeat2->mutable_resources_total())["CPU"] = 1;
  (*heartbeat2->mutable_resources_total())["GPU"] = 10;
  heartbeat2->set_resources_available_changed(true);
  (*heartbeat2->mutable_resources_available())["GPU"] = 5;
  RAY_CHECK_OK(gcs_client_->NodeResources().AsyncReportResourceUsage(
      heartbeat2, [&promise2](Status status) { promise2.set_value(status.ok()); }));
  WaitReady(promise2.get_future(), timeout_ms_);

  resources = global_state_->GetAllResourceUsage();
  resource_usage_batch_data.ParseFromString(*resources.get());
  ASSERT_EQ(resource_usage_batch_data.batch_size(), 1);
  auto resources_data = resource_usage_batch_data.mutable_batch()->at(0);
  ASSERT_EQ(resources_data.resources_total_size(), 2);
  ASSERT_EQ((*resources_data.mutable_resources_total())["CPU"], 1.0);
  ASSERT_EQ((*resources_data.mutable_resources_total())["GPU"], 10.0);
  ASSERT_EQ(resources_data.resources_available_size(), 1);
  ASSERT_EQ((*resources_data.mutable_resources_available())["GPU"], 5.0);

  // Report unchanged resource usage. (Only works with light resource usage report
  // enabled)
  std::promise<bool> promise3;
  auto heartbeat3 = std::make_shared<rpc::ResourcesData>();
  heartbeat3->set_node_id(node_table_data->node_id());
  (*heartbeat3->mutable_resources_available())["CPU"] = 1;
  (*heartbeat3->mutable_resources_available())["GPU"] = 6;
  RAY_CHECK_OK(gcs_client_->NodeResources().AsyncReportResourceUsage(
      heartbeat3, [&promise3](Status status) { promise3.set_value(status.ok()); }));
  WaitReady(promise3.get_future(), timeout_ms_);

  resources = global_state_->GetAllResourceUsage();
  resource_usage_batch_data.ParseFromString(*resources.get());
  ASSERT_EQ(resource_usage_batch_data.batch_size(), 1);
  resources_data = resource_usage_batch_data.mutable_batch()->at(0);
  ASSERT_EQ(resources_data.resources_total_size(), 2);
  ASSERT_EQ((*resources_data.mutable_resources_total())["CPU"], 1.0);
  ASSERT_EQ((*resources_data.mutable_resources_total())["GPU"], 10.0);
  ASSERT_EQ(resources_data.resources_available_size(), 1);
  ASSERT_EQ((*resources_data.mutable_resources_available())["GPU"], 5.0);
}

TEST_P(GlobalStateAccessorTest, TestProfileTable) {
  int profile_count = RayConfig::instance().maximum_profile_table_rows_count() + 1;
  ASSERT_EQ(global_state_->GetAllProfileInfo().size(), 0);
  for (int index = 0; index < profile_count; ++index) {
    auto node_id = NodeID::FromRandom();
    auto profile_table_data = Mocker::GenProfileTableData(node_id);
    std::promise<bool> promise;
    RAY_CHECK_OK(gcs_client_->Stats().AsyncAddProfileData(
        profile_table_data,
        [&promise](Status status) { promise.set_value(status.ok()); }));
    WaitReady(promise.get_future(), timeout_ms_);
  }
  ASSERT_EQ(global_state_->GetAllProfileInfo().size(),
            RayConfig::instance().maximum_profile_table_rows_count());
}

TEST_P(GlobalStateAccessorTest, TestWorkerTable) {
  ASSERT_EQ(global_state_->GetAllWorkerInfo().size(), 0);
  // Add worker info
  auto worker_table_data = Mocker::GenWorkerTableData();
  worker_table_data->mutable_worker_address()->set_worker_id(
      WorkerID::FromRandom().Binary());
  ASSERT_TRUE(global_state_->AddWorkerInfo(worker_table_data->SerializeAsString()));

  // Get worker info
  auto worker_id = WorkerID::FromBinary(worker_table_data->worker_address().worker_id());
  ASSERT_TRUE(global_state_->GetWorkerInfo(worker_id));

  // Add another worker info
  auto another_worker_data = Mocker::GenWorkerTableData();
  another_worker_data->mutable_worker_address()->set_worker_id(
      WorkerID::FromRandom().Binary());
  ASSERT_TRUE(global_state_->AddWorkerInfo(another_worker_data->SerializeAsString()));
  ASSERT_EQ(global_state_->GetAllWorkerInfo().size(), 2);
}

// TODO(sang): Add tests after adding asyncAdd
TEST_P(GlobalStateAccessorTest, TestPlacementGroupTable) {
  ASSERT_EQ(global_state_->GetAllPlacementGroupInfo().size(), 0);
}

INSTANTIATE_TEST_SUITE_P(RedisRemovalTest, GlobalStateAccessorTest,
                         ::testing::Values(false, true));

}  // namespace ray

int main(int argc, char **argv) {
  ray::RayLog::InstallFailureSignalHandler(argv[0]);
  InitShutdownRAII ray_log_shutdown_raii(ray::RayLog::StartRayLog,
                                         ray::RayLog::ShutDownRayLog, argv[0],
                                         ray::RayLogLevel::INFO,
                                         /*log_dir=*/"");
  ::testing::InitGoogleTest(&argc, argv);
  RAY_CHECK(argc == 3);
  ray::TEST_REDIS_SERVER_EXEC_PATH = argv[1];
  ray::TEST_REDIS_CLIENT_EXEC_PATH = argv[2];
  return RUN_ALL_TESTS();
}
<|endoftext|>