Introduce opening book support

My study have shown that it does no harm to use an opening book instead of a search, while the latter being largely deterministic.

This is a primitive version, using a PolyGlot-like book format but with our own Zobrist hash keys, so the probing might be cheap enough to be used elsewhere in search.

The burden is on the creation of the book to provide more diversity to the openings and cover some corner cases, which is another area of research.

No functional changes without a book.

src/Makefile

@@ -38,7 +38,8 @@ PGOBENCH = ./$(EXE) bench
 ### Object files
 OBJS = benchmark.o bitbase.o bitboard.o endgame.o evaluate.o main.o \
 	material.o misc.o movegen.o movepick.o pawns.o position.o psqt.o \
-	search.o thread.o timeman.o tt.o uci.o ucioption.o syzygy/tbprobe.o
+	search.o thread.o timeman.o tt.o uci.o ucioption.o book.o \
+	syzygy/tbprobe.o
 
 ### Establish the operating system name
 KERNEL = $(shell uname -s)

src/book.cpp

@@ -0,0 +1,188 @@
+/*
+  Stockfish, a UCI chess playing engine derived from Glaurung 2.1
+  Copyright (C) 2004-2008 Tord Romstad (Glaurung author)
+  Copyright (C) 2008-2015 Marco Costalba, Joona Kiiski, Tord Romstad
+  Copyright (C) 2015-2019 Marco Costalba, Joona Kiiski, Gary Linscott, Tord Romstad
+
+  Stockfish is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  Stockfish is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "misc.h"
+#include "book.h"
+#include "movegen.h"
+#include <algorithm>
+#include <iostream>
+
+Book Books;
+
+Book::Book()
+{
+    max_book_ply = 400;
+    NumBookEntries = 0;
+    BookEntries = nullptr;
+}
+
+Book::~Book()
+{
+    if (BookEntries != nullptr)
+        delete[] BookEntries;
+}
+
+void Book::init(const std::string& filename)
+{
+    if (filename.length() == 0) return;
+    const char *fnam = filename.c_str();
+
+    if (strcmp(fnam, "<empty>") == 0)
+    {
+        return;
+    }
+
+    FILE *fpt = fopen(fnam, "rb");
+    if (fpt == NULL)
+    {
+        sync_cout << "info string Could not open " << filename << sync_endl;
+        return;
+    }
+
+    if (BookEntries != nullptr)
+    {
+        free(BookEntries);
+        BookEntries = nullptr;
+    }
+
+    fseek(fpt, 0L, SEEK_END);
+    int filesize = ftell(fpt);
+    fseek(fpt, 0L, SEEK_SET);
+
+    NumBookEntries = filesize / sizeof(BookEntry);
+    BookEntries = new BookEntry[NumBookEntries];
+
+    size_t nRead = fread(BookEntries, 1, filesize, fpt);
+    fclose(fpt);
+
+    if (nRead != 1)
+    {
+        free(BookEntries);
+        BookEntries = nullptr;
+        sync_cout << "info string Could not read " << filename << sync_endl;
+        return;
+    }
+    for (size_t i = 0; i < NumBookEntries; i++)
+        byteswap_bookentry(&BookEntries[i]);
+
+    sync_cout << "info string Book loaded: " << filename << " (" << NumBookEntries << " entries)" << sync_endl;
+}
+
+void Book::set_max_ply(int new_max_ply)
+{
+    max_book_ply = new_max_ply;
+}
+
+Move Book::probe_root(Position& pos)
+{
+    if (BookEntries != nullptr && pos.game_ply() < max_book_ply)
+    {
+        int count = 0;
+        size_t index = find_first_entry(pos.key(), count);
+        if (count > 0)
+        {
+            PRNG rng(now());
+            Move move = reconstruct_move(BookEntries[index + rng.rand<unsigned>() % count].move);
+
+            // Add 'special move' flags and verify it is legal
+            for (const auto& m : MoveList<LEGAL>(pos))
+            {
+                if (move == (m.move & (~(3 << 14))))  //  compare with MoveType (bit 14-15)  masked out
+                    return m;
+            }
+        }
+    }
+    return MOVE_NONE;
+}
+
+void Book::probe(Position& pos, std::vector<Move>& bookmoves)
+{
+    if (BookEntries != nullptr && pos.game_ply() < max_book_ply)
+    {
+        int count = 0;
+        size_t index = find_first_entry(pos.key(), count);
+        if (count > 0)
+        {
+            for (int i = 0; i < count; i++)
+                bookmoves.push_back(Move(BookEntries[index + i].move));
+        }
+    }
+}
+
+Move Book::reconstruct_move(uint16_t book_move)
+{
+    Move move = Move(book_move);
+
+    int pt = (move >> 12) & 7;
+    if (pt)
+        return make<PROMOTION>(from_sq(move), to_sq(move), PieceType(pt + 1));
+
+    return move;
+}
+
+size_t Book::find_first_entry(uint64_t key, int& index_count)
+{
+    size_t start = 0;
+    size_t end = NumBookEntries;
+
+    for (;;)
+    {
+        size_t mid = (end + start) / 2;
+
+        if (BookEntries[mid].key < key)
+            start = mid;
+        else
+        {
+            if (BookEntries[mid].key > key)
+                end = mid;
+            else
+            {
+                start = std::max(mid - 4, (size_t)0);
+                end = std::min(mid + 4, NumBookEntries);
+            }
+        }
+
+        if (end - start < 9)
+            break;
+    }
+
+    for (size_t i = start; i < end; i++)
+    {
+        if (key == BookEntries[i].key)
+        {
+            while ((i > 0) && (key == BookEntries[i - 1].key))
+                i--;
+            index_count = 1;
+            end = i;
+            while ((++end < NumBookEntries) && (key == BookEntries[end].key))
+                index_count++;
+            return i;
+        }
+    }
+    return 0;
+}
+
+void Book::byteswap_bookentry(BookEntry *be)
+{
+    be->key = number<uint64_t, BigEndian>(&be->key);
+    be->move = number<uint16_t, BigEndian>(&be->move);
+    be->weight = number<uint16_t, BigEndian>(&be->weight);
+    be->score = number<uint32_t, BigEndian>(&be->score);
+}

src/book.h

@@ -0,0 +1,53 @@
+/*
+  Stockfish, a UCI chess playing engine derived from Glaurung 2.1
+  Copyright (C) 2004-2008 Tord Romstad (Glaurung author)
+  Copyright (C) 2008-2015 Marco Costalba, Joona Kiiski, Tord Romstad
+  Copyright (C) 2015-2019 Marco Costalba, Joona Kiiski, Gary Linscott, Tord Romstad
+
+  Stockfish is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  Stockfish is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef BOOK_H_INCLUDED
+#define BOOK_H_INCLUDED
+
+#include "position.h"
+#include <vector>
+
+class Book
+{
+    typedef struct {
+        uint64_t key;
+        uint16_t move;
+        uint16_t weight;
+        uint32_t score;
+    } BookEntry;
+public:
+    Book();
+    ~Book();
+    void init(const std::string& filename);
+    void set_max_ply(int new_max_ply);
+	Move probe_root(Position& pos);
+	void probe(Position& pos, std::vector<Move>& bookmoves);
+private:
+    size_t find_first_entry(uint64_t key, int& index_count);
+    Move reconstruct_move(uint16_t book_move);
+    size_t NumBookEntries;
+    BookEntry *BookEntries;
+    void byteswap_bookentry(BookEntry *be);
+    int max_book_ply;
+};
+
+extern Book Books;
+
+#endif // #ifndef BOOK_H_INCLUDED

src/main.cpp

@@ -27,6 +27,7 @@
 #include "tt.h"
 #include "uci.h"
 #include "syzygy/tbprobe.h"
+#include "book.h"
 
 namespace PSQT {
   void init();
@@ -44,6 +45,7 @@ int main(int argc, char* argv[]) {
   Search::init();
   Pawns::init();
   Threads.set(Options["Threads"]);
+  Books.init(Options["BookFile"]);
   Search::clear(); // After threads are up
 
   UCI::loop(argc, argv);

src/misc.h

@@ -26,6 +26,7 @@
 #include <ostream>
 #include <string>
 #include <vector>
+#include <cstring>   // For std::memset and std::memcpy
 
 #include "types.h"
 
@@ -111,4 +112,33 @@ namespace WinProcGroup {
   void bindThisThread(size_t idx);
 }
 
+enum { BigEndian, LittleEndian };
+template<typename T, int Half = sizeof(T) / 2, int End = sizeof(T) - 1>
+inline void swap_endian(T& x)
+{
+    static_assert(std::is_unsigned<T>::value, "Argument of swap_endian not unsigned");
+
+    uint8_t tmp, *c = (uint8_t*)&x;
+    for (int i = 0; i < Half; ++i)
+        tmp = c[i], c[i] = c[End - i], c[End - i] = tmp;
+}
+template<> inline void swap_endian<uint8_t>(uint8_t&) {}
+
+template<typename T, int LE> T number(void* addr)
+{
+    static const union { uint32_t i; char c[4]; } Le = { 0x01020304 };
+    static const bool IsLittleEndian = (Le.c[0] == 4);
+
+    T v;
+
+    if ((uintptr_t)addr & (alignof(T) - 1)) // Unaligned pointer (very rare)
+        std::memcpy(&v, addr, sizeof(T));
+    else
+        v = *((T*)addr);
+
+    if (LE != IsLittleEndian)
+        swap_endian(v);
+    return v;
+}
+
 #endif // #ifndef MISC_H_INCLUDED

src/search.cpp

@@ -25,6 +25,7 @@
 #include <iostream>
 #include <sstream>
 
+#include "book.h"
 #include "evaluate.h"
 #include "misc.h"
 #include "movegen.h"
@@ -216,11 +217,24 @@ void MainThread::search() {
   }
   else
   {
-      for (Thread* th : Threads)
-          if (th != this)
-              th->start_searching();
+      Move bookMove = MOVE_NONE;
+
+      if (!Limits.infinite && !Limits.mate)
+          bookMove = Books.probe_root(rootPos);
+
+      if (bookMove && std::count(rootMoves.begin(), rootMoves.end(), bookMove))
+      {
+          for (Thread* th : Threads)
+              std::swap(th->rootMoves[0], *std::find(th->rootMoves.begin(), th->rootMoves.end(), bookMove));
+      }
+      else
+      {
+          for (Thread* th : Threads)
+              if (th != this)
+                  th->start_searching();
 
-      Thread::search(); // Let's start searching!
+          Thread::search(); // Let's start searching!
+      }
   }
 
   // When we reach the maximum depth, we can arrive here without a raise of
@@ -884,6 +898,10 @@ namespace {
                                       contHist,
                                       countermove,
                                       ss->killers);
+
+    std::vector<Move> bookMoves;
+    Books.probe(pos, bookMoves);
+
     value = bestValue; // Workaround a bogus 'uninitialized' warning under gcc
 
     skipQuiets = false;
@@ -907,6 +925,9 @@ namespace {
                                   thisThread->rootMoves.begin() + thisThread->pvLast, move))
           continue;
 
+      if (bookMoves.size() && !std::count(bookMoves.begin(), bookMoves.end(), move & (~(3 << 14))))
+          continue;
+
       ss->moveCount = ++moveCount;
 
       if (rootNode && thisThread == Threads.main() && Time.elapsed() > 3000)