5c nethack

config/default.ini

@@ -107,6 +107,11 @@ vtrace_c_clip = 1.0
 prio_alpha = 0.8
 prio_beta0 = 0.2
 
+# Verb-eps exploration floor (experimental, Nethack; see src/algo.cu). 0 = off.
+verb_eps = 0.0
+verb_eps_anneal_start = 0.4
+verb_eps_anneal_end = 1.0
+
 [sweep]
 metric = score
 metric_distribution = linear

config/nethack.ini

@@ -1,130 +1,113 @@
 [base]
 env_name = nethack
 reset_every_horizon = 0
-async = 1
+async = 0
 
 [vec]
-total_agents = 256
+total_agents = 512
 num_buffers = 4
 #num_threads = 8
 num_threads = 32
-action_mask_size = 690
+action_mask_size = 731
 
 [env]
-gold_coef = 1
-exp_coef = 0.0222526733
-descent_coef = 4.57267237
-xp_coef = 0.648124218
-scout_coef = 0.1
-hp_coef = 0.00617194036
-hunger_coef = 0.01
-illegal_penalty = -0.01
-death_penalty = -0.112676442
-ac_coef = 1.0
-heal_coef = 0.001
-status_coef = 0.0116204191
+gold_coef = 0.021659979757862095
+exp_coef = 0.08341832601491733
+descent_coef = 1.0
+floor_coef = 0.0
+xp_coef = 0.5926087261188399
+scout_coef = 0.025
+death_penalty = -0.8209825710553373
 
 [policy]
 hidden_size = 512
 num_layers = 3
 
 [train]
-#total_timesteps = 166275056
-total_timesteps = 4_000_000_000
-minibatch_size = 16384
-horizon = 512
-learning_rate = 0.00175767997
-gamma = 0.993521929
-gae_lambda = 0.611256123
-replay_ratio = 2.42882919
-clip_coef = 0.478926629
-vf_coef = 2.29309034
-vf_clip_coef = 3.19970345
-max_grad_norm = 1.43588173
-ent_coef = 0.0192690529
-beta1 = 0.9694917707305044
-beta2 = 0.9475724955192064
-eps = 0.0001
-prio_alpha = 0.931300879
-prio_beta0 = 0.885229468
+total_timesteps = 247765094
+minibatch_size = 32768
+horizon = 64
+learning_rate = 0.011854702184553156
+# 5c muon reads train.momentum (NOT beta1); 4.0 fed beta1 as muon momentum
+momentum = 0.805399553172738
+verb_eps = 0.005
+verb_eps_anneal_start = 0.4
+verb_eps_anneal_end = 1.0
+gamma = 0.9973714278257158
+gae_lambda = 0.7339946788354172
+replay_ratio = 1.036717410123433
+clip_coef = 1.0
+vf_coef = 1.8486010633248169
+vf_clip_coef = 4.624607731543118
+max_grad_norm = 0.902165688849037
+ent_coef = 0.008389210353641681
+beta1 = 0.805399553172738
+beta2 = 0.999139492311758
+eps = 1e-14
+vtrace_rho_clip = 2.770648671516198
+vtrace_c_clip = 4.5026502564349595
+prio_alpha = 0.6754855924309153
+prio_beta0 = 0.8672463883417074
 
 [sweep]
-metric = max_depth
+metric = score
 
 [sweep.train.total_timesteps]
 distribution = log_normal
 min = 1.5e8
 max = 4e8
 scale = time
 
-[sweep.env.gold_coef]
-distribution = log_normal
-min = 0.001
-max = 1.0
-scale = auto
-
-[sweep.env.exp_coef]
-distribution = log_normal
-min = 0.01
-max = 10.0
-scale = auto
-
-[sweep.env.descent_coef]
+[sweep.train.verb_eps]
 distribution = log_normal
-min = 0.5
-max = 10.0
+min = 0.0005
+max = 0.02
 scale = auto
 
-[sweep.env.xp_coef]
-distribution = log_normal
-min = 0.05
-max = 1.0
+[sweep.train.verb_eps_anneal_start]
+distribution = uniform
+min = 0.2
+max = 0.8
 scale = auto
 
-[sweep.env.scout_coef]
+[sweep.env.exp_coef]
 distribution = log_normal
-min = 0.0001
+min = 0.01
 max = 0.1
 scale = auto
 
-[sweep.env.hp_coef]
+[sweep.env.gold_coef]
 distribution = log_normal
-min = 0.0005
-max = 0.06
+min = 0.001
+max = 0.1
 scale = auto
 
-[sweep.env.hunger_coef]
-distribution = log_normal
-min = 0.01
-max = 0.25
+[sweep.env.descent_coef]
+distribution = uniform
+min = 0.0
+max = 1.0
 scale = auto
 
-[sweep.env.illegal_penalty]
+[sweep.env.floor_coef]
 distribution = uniform
-min = -0.01
-max = 0.0
+min = 0.0
+max = 1.0
 scale = auto
 
-[sweep.env.death_penalty]
+[sweep.env.xp_coef]
 distribution = uniform
-min = -1.0
-max = 0.0
+min = 0.0
+max = 1.0
 scale = auto
 
-[sweep.env.heal_coef]
+[sweep.env.scout_coef]
 distribution = log_normal
 min = 0.001
 max = 0.1
 scale = auto
 
-[sweep.env.status_coef]
-distribution = log_normal
-min = 0.01
-max = 1.0
-scale = auto
-
-[sweep.env.ac_coef]
-distribution = log_normal
-min = 0.01
-max = 0.5
+[sweep.env.death_penalty]
+distribution = uniform
+min = -1.0
+max = 0.0
 scale = auto

ocean/nethack/macros.h

@@ -26,6 +26,17 @@ static int nethack_msg_contains(const Nethack* env, const char* needle) {
     return strstr(buf, needle) != NULL;
 }
 
+// first integer after `needle`, 0 if absent ("<Shk> offers [only] N gold ...")
+static long nethack_msg_number_after(const Nethack* env, const char* needle) {
+    char buf[NLE_MESSAGE_SIZE + 1];
+    memcpy(buf, env->message, NLE_MESSAGE_SIZE);
+    buf[NLE_MESSAGE_SIZE] = '\0';
+    const char* p = strstr(buf, needle);
+    if (!p) return 0;
+    while (*p && (*p < '0' || *p > '9')) p++;
+    return strtol(p, NULL, 10);
+}
+
 // parse a getobj bracket list ("[b-d f or ?*]") into cand[]; returns count
 static int nethack_parse_candidates(const Nethack* env, char* cand, int cap) {
     const unsigned char* m = env->message;
@@ -93,6 +104,13 @@ static int nethack_handle_prompts(Nethack* env) {
         int commit = yn && nethack_msg_contains(env, "[yn")
                         && !nethack_msg_contains(env, "no return")
                         && !nethack_msg_contains(env, "eally attack");
+        // shopkeeper's "<Shk> offers N gold pieces for your X.  Sell it?" is
+        // ynaq-rendered, so commit already accepts it — count the conversion
+        if (commit && nethack_msg_contains(env, "gold piece")
+                   && nethack_msg_contains(env, "Sell")) {
+            env->stats.sells++;
+            env->stats.sale_gold += nethack_msg_number_after(env, "offers");
+        }
         env->obs.action = ring ? 'r' : (commit ? 'y' : 27);
         env->ctx = nle_step(env->ctx, &env->obs);
     }