diff --git a/lua/relative-toggle/init.lua b/lua/relative-toggle/init.lua
index 52b3cc2..5d38fe6 100644
--- a/lua/relative-toggle/init.lua
+++ b/lua/relative-toggle/init.lua
@@ -14,27 +14,35 @@ local function create_autocmd(event, opts)
     end
 end
 
+-- to keep the combination of number and relativenumber
+local current_number = vim.o.number
+
 ---@param relative boolean #whether relativenumber should be set
 ---@param redraw boolean #whether to redraw the screen
 local function set_relativenumber(relative, redraw)
+    -- ignore for buffer or window with these off by default
+    if not vim.o.number and not vim.o.relativenumber then
+        return
+    end
+
     local in_insert_mode = vim.api.nvim_get_mode().mode == "i"
 
-    if vim.o.number then
-        vim.opt.relativenumber = relative and not in_insert_mode
+    vim.opt.number = not relative or in_insert_mode or current_number
+    vim.opt.relativenumber = relative and not in_insert_mode
 
-        if redraw then
-            vim.cmd "redraw"
-        end
+    if redraw then
+        vim.cmd "redraw"
     end
 end
 
 function M.setup(user_config)
-    -- load user config
-    config:extend(user_config)
-
     local events = config.events
     local augroup = vim.api.nvim_create_augroup("relative-toggle", {})
 
+    vim.opt.relativenumber = true
+
+    config:extend(user_config)
+
     create_autocmd(events.on, {
         pattern = config.pattern,
         group = augroup,