Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
R
RL-Parsing
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Franck Dary
RL-Parsing
Commits
9547a7de
Commit
9547a7de
authored
4 years ago
by
Franck Dary
Browse files
Options
Downloads
Patches
Plain Diff
In RL, consider non appliable action
parent
d36e1f08
No related branches found
No related tags found
No related merge requests found
Changes
3
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
Decode.py
+3
-3
3 additions, 3 deletions
Decode.py
Rl.py
+11
-9
11 additions, 9 deletions
Rl.py
Train.py
+29
-7
29 additions, 7 deletions
Train.py
with
43 additions
and
19 deletions
Decode.py
+
3
−
3
View file @
9547a7de
...
...
@@ -57,15 +57,15 @@ def decodeModel(ts, strat, config, network, dicts, debug) :
with
torch
.
no_grad
():
while
moved
:
features
=
extractFeatures
(
dicts
,
config
).
unsqueeze
(
0
).
to
(
decodeDevice
)
output
=
torch
.
nn
.
functional
.
softmax
(
network
(
features
)
,
dim
=
1
)
scores
=
sorted
([[
"
%.2f
"
%
float
(
output
[
0
][
index
]),
ts
[
index
].
appliable
(
config
),
ts
[
index
].
name
]
for
index
in
range
(
len
(
ts
))])[::
-
1
]
output
=
network
(
features
)
scores
=
sorted
([[
float
(
output
[
0
][
index
]),
ts
[
index
].
appliable
(
config
),
ts
[
index
].
name
]
for
index
in
range
(
len
(
ts
))])[::
-
1
]
candidates
=
[[
cand
[
0
],
cand
[
2
]]
for
cand
in
scores
if
cand
[
1
]]
if
len
(
candidates
)
==
0
:
break
candidate
=
candidates
[
0
][
1
]
if
debug
:
config
.
printForDebug
(
sys
.
stderr
)
print
(
"
"
.
join
([
"
%s%
s
:%s
"
%
(
"
*
"
if
score
[
1
]
else
"
"
,
score
[
0
],
score
[
2
])
for
score
in
scores
])
+
"
\n
"
+
(
"
-
"
*
80
)
+
"
\n
"
,
file
=
sys
.
stderr
)
print
(
"
"
.
join
([
"
%s%
.2f
:%s
"
%
(
"
*
"
if
score
[
1
]
else
"
"
,
score
[
0
],
score
[
2
])
for
score
in
scores
])
+
"
\n
"
+
(
"
-
"
*
80
)
+
"
\n
"
,
file
=
sys
.
stderr
)
moved
=
applyTransition
(
ts
,
strat
,
config
,
candidate
)
EOS
.
apply
(
config
)
...
...
This diff is collapsed.
Click to expand it.
Rl.py
+
11
−
9
View file @
9547a7de
...
...
@@ -12,13 +12,16 @@ class ReplayMemory() :
self
.
newStates
=
torch
.
zeros
(
capacity
,
stateSize
,
dtype
=
torch
.
long
,
device
=
getDevice
())
self
.
actions
=
torch
.
zeros
(
capacity
,
1
,
dtype
=
torch
.
long
,
device
=
getDevice
())
self
.
rewards
=
torch
.
zeros
(
capacity
,
1
,
device
=
getDevice
())
self
.
noNewStates
=
torch
.
zeros
(
capacity
,
dtype
=
torch
.
bool
,
device
=
getDevice
())
self
.
position
=
0
self
.
nbPushed
=
0
def
push
(
self
,
state
,
action
,
newState
,
reward
)
:
self
.
states
[
self
.
position
]
=
state
self
.
actions
[
self
.
position
]
=
action
if
newState
is
not
None
:
self
.
newStates
[
self
.
position
]
=
newState
self
.
noNewStates
[
self
.
position
]
=
newState
is
None
self
.
rewards
[
self
.
position
]
=
reward
self
.
position
=
(
self
.
position
+
1
)
%
self
.
capacity
self
.
nbPushed
+=
1
...
...
@@ -26,7 +29,7 @@ class ReplayMemory() :
def
sample
(
self
,
batchSize
)
:
start
=
random
.
randint
(
0
,
len
(
self
)
-
batchSize
)
end
=
start
+
batchSize
return
self
.
states
[
start
:
end
],
self
.
actions
[
start
:
end
],
self
.
newStates
[
start
:
end
],
self
.
rewards
[
start
:
end
]
return
self
.
states
[
start
:
end
],
self
.
actions
[
start
:
end
],
self
.
newStates
[
start
:
end
],
self
.
noNewStates
[
start
:
end
],
self
.
rewards
[
start
:
end
]
def
__len__
(
self
):
return
min
(
self
.
nbPushed
,
self
.
capacity
)
...
...
@@ -36,30 +39,29 @@ class ReplayMemory() :
def
selectAction
(
network
,
state
,
ts
,
config
,
missingLinks
,
probaRandom
,
probaOracle
)
:
sample
=
random
.
random
()
if
sample
<
probaRandom
:
candidates
=
[
trans
for
trans
in
ts
if
trans
.
appliable
(
config
)]
return
candidates
[
random
.
randrange
(
len
(
candidates
))]
if
len
(
candidates
)
>
0
else
None
return
ts
[
random
.
randrange
(
len
(
ts
))]
elif
sample
<
probaRandom
+
probaOracle
:
candidates
=
sorted
([[
trans
.
getOracleScore
(
config
,
missingLinks
),
trans
]
for
trans
in
ts
if
trans
.
appliable
(
config
)])
return
candidates
[
0
][
1
]
if
len
(
candidates
)
>
0
else
None
else
:
with
torch
.
no_grad
()
:
output
=
network
(
torch
.
stack
([
state
]))
candidates
=
sorted
([[
ts
[
index
].
appliable
(
config
),
"
%.2f
"
%
float
(
output
[
0
][
index
]),
ts
[
index
]]
for
index
in
range
(
len
(
ts
))])[::
-
1
]
candidates
=
[
cand
[
2
]
for
cand
in
candidates
if
cand
[
0
]]
return
candidates
[
0
]
if
len
(
candidates
)
>
0
else
None
predIndex
=
int
(
torch
.
argmax
(
output
))
return
ts
[
predIndex
]
################################################################################
################################################################################
def
optimizeModel
(
batchSize
,
policy_net
,
target_net
,
memory
,
optimizer
)
:
gamma
=
0.9
99
gamma
=
0.9
if
len
(
memory
)
<
batchSize
:
return
0.0
states
,
actions
,
nextStates
,
rewards
=
memory
.
sample
(
batchSize
)
states
,
actions
,
nextStates
,
noNextStates
,
rewards
=
memory
.
sample
(
batchSize
)
predictedQ
=
policy_net
(
states
).
gather
(
1
,
actions
)
nextQ
=
target_net
(
nextStates
).
max
(
1
)[
0
].
detach
().
unsqueeze
(
0
)
nextQ
=
torch
.
transpose
(
nextQ
,
0
,
1
)
nextQ
[
noNextStates
]
=
0.0
expectedReward
=
gamma
*
nextQ
+
rewards
...
...
This diff is collapsed.
Click to expand it.
Train.py
+
29
−
7
View file @
9547a7de
...
...
@@ -140,13 +140,21 @@ def trainModelRl(debug, modelDir, filename, nbIter, batchSize, devFile, transiti
bestLoss
=
None
bestScore
=
None
sentences
=
copy
.
deepcopy
(
sentencesOriginal
)
nbExByEpoch
=
sum
(
map
(
len
,
sentences
))
sentIndex
=
0
for
epoch
in
range
(
1
,
nbIter
+
1
)
:
i
=
0
totalLoss
=
0.0
while
True
:
if
sentIndex
>=
len
(
sentences
)
:
sentences
=
copy
.
deepcopy
(
sentencesOriginal
)
for
sentIndex
in
range
(
len
(
sentences
))
:
random
.
shuffle
(
sentences
)
sentIndex
=
0
if
not
silent
:
print
(
"
Curent epoch %6.2f%%
"
%
(
100.0
*
sentIndex
/
len
(
sentences
)
),
end
=
"
\r
"
,
file
=
sys
.
stderr
)
print
(
"
Curent epoch %6.2f%%
"
%
(
100.0
*
i
/
nbExByEpoch
),
end
=
"
\r
"
,
file
=
sys
.
stderr
)
sentence
=
sentences
[
sentIndex
]
sentence
.
moveWordIndex
(
0
)
state
=
Features
.
extractFeaturesPosExtended
(
dicts
,
sentence
).
to
(
getDevice
())
...
...
@@ -168,14 +176,22 @@ def trainModelRl(debug, modelDir, filename, nbIter, batchSize, devFile, transiti
if
action
is
None
:
break
appliable
=
action
.
appliable
(
sentence
)
# Reward for doing an illegal action
reward
=
-
3.0
if
appliable
:
reward
=
-
1.0
*
action
.
getOracleScore
(
sentence
,
missingLinks
)
reward
=
torch
.
FloatTensor
([
reward
]).
to
(
getDevice
())
newState
=
None
if
appliable
:
applyTransition
(
transitionSet
,
strategy
,
sentence
,
action
.
name
)
newState
=
Features
.
extractFeaturesPosExtended
(
dicts
,
sentence
).
to
(
getDevice
())
if
memory
is
None
:
memory
=
ReplayMemory
(
1
000
,
state
.
numel
())
memory
=
ReplayMemory
(
5
000
,
state
.
numel
())
memory
.
push
(
state
,
torch
.
LongTensor
([
transitionSet
.
index
(
action
)]).
to
(
getDevice
()),
newState
,
reward
)
state
=
newState
if
i
%
batchSize
==
0
:
...
...
@@ -185,6 +201,12 @@ def trainModelRl(debug, modelDir, filename, nbIter, batchSize, devFile, transiti
target_net
.
eval
()
policy_net
.
train
()
i
+=
1
if
state
is
None
:
break
if
i
>=
nbExByEpoch
:
break
sentIndex
+=
1
bestLoss
,
bestScore
=
evalModelAndSave
(
debug
,
policy_net
,
dicts
,
modelDir
,
devFile
,
bestLoss
,
totalLoss
,
bestScore
,
epoch
,
nbIter
)
################################################################################
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment