You would have to alter the query by adding a meta_query for the user field.
$meta_query = array(
array(
'key' => 'user_field_name',
// user field is stored as a serialized array
// of string User IDs
// value must be enclosed in quotes
// compare must be LIKE
'value' => '"'.$user_id.'"',
'compare' => 'LIKE'
)
);
$query->set('meta_query', $meta_query);